diff --git a/.github/DEVELOPER.md b/.github/DEVELOPER.md index 5dd5dfa9..a82ef06d 100644 --- a/.github/DEVELOPER.md +++ b/.github/DEVELOPER.md @@ -55,6 +55,13 @@ You can also run `make coverage` to generate a coverage report. ## Running BDD tests +### Dependencies + +1. You will need an existing EKS cluster running with the connection details exported into a kube config file. +2. [Keikoproj Minion-Manager](https://github.com/keikoproj/minion-manager) must also be running in the cluster +3. Instance Manager needs to be started outside of the bdd test suite + + Export some variables and run `make bdd` to run a functional e2e test. ### Example @@ -96,3 +103,5 @@ testing: warning: no tests to run PASS ok github.com/keikoproj/instance-manager/test-bdd 1362.336s [no tests to run] ``` + +Note: If your test cluster uses `InstanceGroups` to run core components, annotating the namespace with `instancemgr.keikoproj.io/config-excluded="true"` can help prevent unexpected disruption. diff --git a/api/v1alpha1/instancegroup_types.go b/api/v1alpha1/instancegroup_types.go index beaebf0d..83340026 100644 --- a/api/v1alpha1/instancegroup_types.go +++ b/api/v1alpha1/instancegroup_types.go @@ -47,8 +47,9 @@ const ( ReconcileModified ReconcileState = "ReconcileModified" // End States - ReconcileReady ReconcileState = "Ready" - ReconcileErr ReconcileState = "Error" + ReconcileLocked ReconcileState = "Locked" + ReconcileReady ReconcileState = "Ready" + ReconcileErr ReconcileState = "Error" // Userdata bootstrap stages PreBootstrapStage = "PreBootstrap" @@ -76,6 +77,8 @@ const ( HostPlacementTenancyType = "host" DefaultPlacementTenancyType = "default" DedicatedPlacementTenancyType = "dedicated" + + ImageLatestValue = "latest" ) type ContainerRuntime string @@ -87,6 +90,8 @@ const ( DockerRuntime ContainerRuntime = "dockerd" ContainerDRuntime ContainerRuntime = "containerd" + + UpgradeLockedAnnotationKey = "instancemgr.keikoproj.io/lock-upgrades" ) var ( @@ -392,6 +397,15 @@ func (ig *InstanceGroup) GetUpgradeStrategy() *AwsUpgradeStrategy { func (ig *InstanceGroup) SetUpgradeStrategy(strategy AwsUpgradeStrategy) { ig.Spec.AwsUpgradeStrategy = strategy } +func (ig *InstanceGroup) Locked() bool { + annotations := ig.GetAnnotations() + if val, ok := annotations[UpgradeLockedAnnotationKey]; ok { + if strings.EqualFold(val, "true") { + return true + } + } + return false +} func (s *EKSSpec) Validate() error { var ( @@ -521,7 +535,6 @@ func (c *EKSConfiguration) Validate() error { c.SuspendedProcesses = processes } - if c.BootstrapOptions != nil { if c.BootstrapOptions.ContainerRuntime != "" && !contains(AllowedContainerRuntimes, c.BootstrapOptions.ContainerRuntime) { return errors.Errorf("validation failed, 'bootstrapOptions.containerRuntime' must be one of %+v", AllowedContainerRuntimes) diff --git a/api/v1alpha1/instancegroup_types_test.go b/api/v1alpha1/instancegroup_types_test.go index 02a787e9..4d94d999 100644 --- a/api/v1alpha1/instancegroup_types_test.go +++ b/api/v1alpha1/instancegroup_types_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/aws/aws-sdk-go/aws" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type EksUnitTest struct { @@ -117,13 +118,13 @@ func TestInstanceGroupSpecValidate(t *testing.T) { MinSize: 1, Type: "LaunchTemplate", EKSConfiguration: &EKSConfiguration{ - BootstrapOptions: &BootstrapOptions{ContainerRuntime: "foo"}, - EksClusterName: "my-eks-cluster", - NodeSecurityGroups: []string{"sg-123456789"}, - Image: "ami-12345", - InstanceType: "m5.large", - KeyPairName: "thisShouldBeOptional", - Subnets: []string{"subnet-1111111", "subnet-222222"}, + BootstrapOptions: &BootstrapOptions{ContainerRuntime: "foo"}, + EksClusterName: "my-eks-cluster", + NodeSecurityGroups: []string{"sg-123456789"}, + Image: "ami-12345", + InstanceType: "m5.large", + KeyPairName: "thisShouldBeOptional", + Subnets: []string{"subnet-1111111", "subnet-222222"}, }, }, nil, nil), }, @@ -353,6 +354,41 @@ func TestInstanceGroupSpecValidate(t *testing.T) { } } +func TestLockedAnnotation(t *testing.T) { + tests := []struct { + name string + annotation string + expected bool + }{ + { + name: "Locked", + annotation: "true", + expected: true, + }, + { + name: "Unlocked", + annotation: "false", + expected: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testIg := &InstanceGroup{ + ObjectMeta: v1.ObjectMeta{ + Annotations: map[string]string{ + UpgradeLockedAnnotationKey: test.annotation, + }, + }, + } + res := testIg.Locked() + if res != test.expected { + t.Errorf("%v: got %v, expected %v", test.name, res, test.expected) + } + }) + } +} + func basicFargateSpec() *EKSFargateSpec { return &EKSFargateSpec{ ClusterName: "", diff --git a/controllers/instancegroup_controller.go b/controllers/instancegroup_controller.go index 92b7ee5a..e6f57ff4 100644 --- a/controllers/instancegroup_controller.go +++ b/controllers/instancegroup_controller.go @@ -246,7 +246,7 @@ func (r *InstanceGroupReconciler) IsNamespaceAnnotated(namespace, key, value str } annotations := unstructuredNamespace.GetAnnotations() - if kubeprovider.HasAnnotation(annotations, key, value) { + if kubeprovider.HasAnnotationWithValue(annotations, key, value) { return true } } diff --git a/controllers/interface.go b/controllers/interface.go index 52f35886..c0159ef5 100644 --- a/controllers/interface.go +++ b/controllers/interface.go @@ -16,6 +16,7 @@ type CloudDeployer interface { GetState() v1alpha.ReconcileState // Gets the current state type of the instance group SetState(v1alpha.ReconcileState) // Sets the current state of the instance group IsReady() bool // Returns true if state is Ready + Locked() bool // Returns true if instanceGroup is locked } func HandleReconcileRequest(d CloudDeployer) error { @@ -54,6 +55,11 @@ func HandleReconcileRequest(d CloudDeployer) error { // CRUD Nodes Upgrade Strategy if d.GetState() == v1alpha.ReconcileInitUpgrade { + // Locked + if d.Locked() { + d.SetState(v1alpha.ReconcileLocked) + return nil + } err = d.UpgradeNodes() if err != nil { return err @@ -67,12 +73,18 @@ func HandleReconcileRequest(d CloudDeployer) error { // Bootstrap Nodes if d.IsReady() { + err = d.BootstrapNodes() if err != nil { return err } if d.GetState() == v1alpha.ReconcileInitUpgrade { + // Locked + if d.Locked() { + d.SetState(v1alpha.ReconcileLocked) + return nil + } err = d.UpgradeNodes() if err != nil { return err diff --git a/controllers/providers/aws/aws.go b/controllers/providers/aws/aws.go index 53d70568..8896885f 100644 --- a/controllers/providers/aws/aws.go +++ b/controllers/providers/aws/aws.go @@ -30,6 +30,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/pkg/errors" ctrl "sigs.k8s.io/controller-runtime" ) @@ -55,6 +56,7 @@ const ( DescribeLaunchTemplateVersionsTTL time.Duration = 60 * time.Second DescribeInstanceTypesTTL time.Duration = 24 * time.Hour DescribeInstanceTypeOfferingTTL time.Duration = 1 * time.Hour + GetParameterTTL time.Duration = 1 * time.Hour CacheBackgroundPruningInterval time.Duration = 1 * time.Hour CacheMaxItems int64 = 250 @@ -117,6 +119,7 @@ type AwsWorker struct { EksClient eksiface.EKSAPI IamClient iamiface.IAMAPI Ec2Client ec2iface.EC2API + SsmClient ssmiface.SSMAPI Ec2Metadata *ec2metadata.EC2Metadata Parameters map[string]interface{} } @@ -246,10 +249,9 @@ func GetScalingConfigName(group *autoscaling.Group) string { } func GetInstanceTypeNetworkInfo(instanceTypes []*ec2.InstanceTypeInfo, instanceType string) *ec2.NetworkInfo { - for _, instanceTypeInfo := range instanceTypes { - if aws.StringValue(instanceTypeInfo.InstanceType) == instanceType { - return instanceTypeInfo.NetworkInfo - } + i := GetInstanceTypeInfo(instanceTypes, instanceType) + if i != nil { + return i.NetworkInfo } return nil } @@ -262,3 +264,11 @@ func GetInstanceTypeInfo(instanceTypes []*ec2.InstanceTypeInfo, instanceType str } return nil } + +func GetInstanceTypeArchitectures(instanceTypes []*ec2.InstanceTypeInfo, instanceType string) []string { + i := GetInstanceTypeInfo(instanceTypes, instanceType) + if i != nil { + return aws.StringValueSlice((*i).ProcessorInfo.SupportedArchitectures) + } + return nil +} diff --git a/controllers/providers/aws/ssm.go b/controllers/providers/aws/ssm.go new file mode 100644 index 00000000..f797beb4 --- /dev/null +++ b/controllers/providers/aws/ssm.go @@ -0,0 +1,72 @@ +package aws + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/keikoproj/aws-sdk-go-cache/cache" + "github.com/keikoproj/instance-manager/controllers/common" +) + +type architectureMap map[string]string + +const ( + EksOptimisedAmiPath = "/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id" + EksOptimisedAmazonLinux2Arm64 = "/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id" + EksOptimisedBottlerocket = "/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id" + EksOptimisedBottlerocketArm64 = "/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id" + EksOptimisedWindowsCore = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id" + EksOptimisedWindowsFull = "/aws/service/ami-windows-latest/Windows_Server-2019-English-Full-EKS_Optimized-%s/image_id" +) + +var ( + EksAmis = map[string]architectureMap{ + "amazonlinux2": architectureMap{ + "x86_64": EksOptimisedAmiPath, + "arm64": EksOptimisedAmazonLinux2Arm64, + }, + "bottlerocket": architectureMap{ + "x86_64": EksOptimisedBottlerocket, + "arm64": EksOptimisedBottlerocketArm64, + }, + "windows": architectureMap{ + "x86_64": EksOptimisedWindowsCore, + }, + } +) + +func GetAwsSsmClient(region string, cacheCfg *cache.Config, maxRetries int, collector *common.MetricsCollector) ssmiface.SSMAPI { + config := aws.NewConfig().WithRegion(region).WithCredentialsChainVerboseErrors(true) + config = request.WithRetryer(config, NewRetryLogger(maxRetries, collector)) + sess, err := session.NewSession(config) + if err != nil { + panic(err) + } + cache.AddCaching(sess, cacheCfg) + cacheCfg.SetCacheTTL("ssm", "GetParameter", GetParameterTTL) + sess.Handlers.Complete.PushFront(func(r *request.Request) { + ctx := r.HTTPRequest.Context() + log.V(1).Info("AWS API call", + "cacheHit", cache.IsCacheHit(ctx), + "service", r.ClientInfo.ServiceName, + "operation", r.Operation.Name, + ) + }) + return ssm.New(sess) +} + +func (w *AwsWorker) GetEksLatestAmi(OSFamily string, arch string, kubernetesVersion string) (string, error) { + input := &ssm.GetParameterInput{ + Name: aws.String(fmt.Sprintf(EksAmis[OSFamily][arch], kubernetesVersion)), + } + + output, err := w.SsmClient.GetParameter(input) + if err != nil { + return "", err + } + return aws.StringValue(output.Parameter.Value), nil +} diff --git a/controllers/providers/kubernetes/crd.go b/controllers/providers/kubernetes/crd.go index dcb4f5ae..60361849 100644 --- a/controllers/providers/kubernetes/crd.go +++ b/controllers/providers/kubernetes/crd.go @@ -329,7 +329,7 @@ func GetResources(kube dynamic.Interface, instanceGroup *v1alpha1.InstanceGroup, annotations := ru.GetAnnotations() - if HasAnnotation(annotations, OwnershipAnnotationKey, OwnershipAnnotationValue) && HasAnnotation(annotations, ScopeAnnotationKey, status.GetActiveScalingGroupName()) { + if HasAnnotationWithValue(annotations, OwnershipAnnotationKey, OwnershipAnnotationValue) && HasAnnotationWithValue(annotations, ScopeAnnotationKey, status.GetActiveScalingGroupName()) { if IsPathValue(ru, statusJSONPath, completedStatus) || IsPathValue(ru, statusJSONPath, errorStatus) { // if resource is not completed and not failed, it must be still active inactiveResources = append(inactiveResources, ru) diff --git a/controllers/providers/kubernetes/utils.go b/controllers/providers/kubernetes/utils.go index fbba540f..82c1ee3d 100644 --- a/controllers/providers/kubernetes/utils.go +++ b/controllers/providers/kubernetes/utils.go @@ -116,7 +116,14 @@ func AddAnnotation(u *unstructured.Unstructured, key, value string) { u.SetAnnotations(annotations) } -func HasAnnotation(annotations map[string]string, key, value string) bool { +func HasAnnotation(annotations map[string]string, key string) bool { + if _, ok := annotations[key]; ok { + return true + } + return false +} + +func HasAnnotationWithValue(annotations map[string]string, key, value string) bool { if val, ok := annotations[key]; ok { if strings.EqualFold(val, value) { return true diff --git a/controllers/providers/kubernetes/utils_test.go b/controllers/providers/kubernetes/utils_test.go new file mode 100644 index 00000000..f42d505c --- /dev/null +++ b/controllers/providers/kubernetes/utils_test.go @@ -0,0 +1,103 @@ +package kubernetes + +import ( + "testing" +) + +func TestHasAnnotation(t *testing.T) { + annotations := map[string]string{ + "foo": "bar", + "baz": "", + } + tests := []struct { + name string + annotations map[string]string + key string + expected bool + }{ + { + name: "present with value", + annotations: annotations, + key: "foo", + expected: true, + }, + { + name: "present with no value", + annotations: annotations, + key: "baz", + expected: true, + }, + { + name: "absent", + annotations: annotations, + key: "missing", + expected: false, + }, + } + + for _, tc := range tests { + result := HasAnnotation(tc.annotations, tc.key) + if result != tc.expected { + t.Fail() + } + } + +} + +func TestHasAnnotationWithValue(t *testing.T) { + annotations := map[string]string{ + "foo": "bar", + "baz": "", + } + tests := []struct { + name string + annotations map[string]string + key string + value string + expected bool + }{ + { + name: "present with value expecting value", + annotations: annotations, + key: "foo", + value: "bar", + expected: true, + }, + { + name: "present with value expecting no value", + annotations: annotations, + key: "foo", + value: "", + expected: false, + }, + { + name: "present with no value expecting no value", + annotations: annotations, + key: "baz", + value: "", + expected: true, + }, + { + name: "present with no value expecting value", + annotations: annotations, + key: "baz", + value: "boop", + expected: false, + }, + { + name: "absent", + annotations: annotations, + key: "missing", + value: "", + expected: false, + }, + } + + for _, tc := range tests { + result := HasAnnotationWithValue(tc.annotations, tc.key, tc.value) + if result != tc.expected { + t.Fatalf("Unexpected result %v. expected %v from %s", result, tc.expected, tc.name) + } + } + +} diff --git a/controllers/provisioners/config_test.go b/controllers/provisioners/config_test.go index ffaa9327..41397d09 100644 --- a/controllers/provisioners/config_test.go +++ b/controllers/provisioners/config_test.go @@ -957,6 +957,7 @@ func TestIsRetryable(t *testing.T) { {state: v1alpha1.ReconcileErr, expectedRetryable: false}, {state: v1alpha1.ReconcileReady, expectedRetryable: false}, {state: v1alpha1.ReconcileDeleted, expectedRetryable: false}, + {state: v1alpha1.ReconcileLocked, expectedRetryable: false}, {state: v1alpha1.ReconcileDeleting, expectedRetryable: true}, {state: v1alpha1.ReconcileInit, expectedRetryable: true}, {state: v1alpha1.ReconcileInitCreate, expectedRetryable: true}, diff --git a/controllers/provisioners/eks/cloud.go b/controllers/provisioners/eks/cloud.go index b4357dbd..4ed8cfe0 100644 --- a/controllers/provisioners/eks/cloud.go +++ b/controllers/provisioners/eks/cloud.go @@ -177,6 +177,16 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error { } state.SetInstanceTypeInfo(instanceTypes) + if strings.EqualFold(configuration.Image, v1alpha1.ImageLatestValue) { + latestAmiId, err := ctx.GetEksLatestAmi() + if err != nil { + return errors.Wrap(err, "failed to discover latest AMI ID") + } + configuration.Image = latestAmiId + ctx.Log.V(4).Info("Updating Image ID with latest", "ami_id", latestAmiId) + } + + // All information needed to creating the scaling group must happen before this line. // find all owned scaling groups ownedScalingGroups := ctx.findOwnedScalingGroups(scalingGroups) state.SetOwnedScalingGroups(ownedScalingGroups) @@ -207,6 +217,7 @@ func (ctx *EksInstanceGroupContext) CloudDiscovery() error { if err != nil { return errors.Wrap(err, "failed to describe lifecycle hooks") } + // update status with scaling group info status.SetActiveScalingGroupName(asgName) status.SetCurrentMin(int(aws.Int64Value(targetScalingGroup.MinSize))) @@ -444,8 +455,7 @@ func subFamilyFlexiblePool(offerings []*ec2.InstanceTypeOffering, typeInfo []*ec continue } - - if !common.StringSliceContains(desiredArchs, supportedArchs){ + if !common.StringSliceContains(desiredArchs, supportedArchs) { continue } diff --git a/controllers/provisioners/eks/cloud_test.go b/controllers/provisioners/eks/cloud_test.go index 02bdbafd..80218db2 100644 --- a/controllers/provisioners/eks/cloud_test.go +++ b/controllers/provisioners/eks/cloud_test.go @@ -42,9 +42,10 @@ func TestCloudDiscoveryPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) state := ctx.GetDiscoveredState() status := ig.GetStatus() @@ -124,9 +125,10 @@ func TestCloudDiscoveryWithTemplatePositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) state := ctx.GetDiscoveredState() status := ig.GetStatus() @@ -223,7 +225,6 @@ func TestDeriveSubFamilyFlexiblePool(t *testing.T) { MockInstanceTypeInfo{"a5i.large", 1, 100, "amd64"}, MockInstanceTypeInfo{"a5g.large", 1, 100, "arm64"}, MockInstanceTypeInfo{"a5a.large", 1, 100, "amd64"}, - ) expectedPool := make(map[string][]InstanceSpec, 0) @@ -341,9 +342,10 @@ func TestCloudDiscoveryExistingRole(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) configuration := ig.GetEKSConfiguration() state := ctx.GetDiscoveredState() @@ -375,9 +377,10 @@ func TestCloudDiscoverySpotPrice(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) status := ig.GetStatus() configuration := ig.GetEKSConfiguration() @@ -453,9 +456,10 @@ func TestLaunchConfigDeletion(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) configuration := ig.GetEKSConfiguration() diff --git a/controllers/provisioners/eks/create_test.go b/controllers/provisioners/eks/create_test.go index 9ec83627..cd371812 100644 --- a/controllers/provisioners/eks/create_test.go +++ b/controllers/provisioners/eks/create_test.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" "github.com/keikoproj/instance-manager/api/v1alpha1" "github.com/onsi/gomega" @@ -40,9 +41,10 @@ func TestCreateManagedRolePositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) state := ctx.GetDiscoveredState() state.SetCluster(MockEksCluster("1.15")) @@ -80,9 +82,10 @@ func TestCreateLaunchConfigurationPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) iamMock.Role = &iam.Role{ @@ -111,9 +114,10 @@ func TestCreateLaunchTemplatePositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) iamMock.Role = &iam.Role{ @@ -144,9 +148,10 @@ func TestCreateScalingGroupPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // skip role creation @@ -187,9 +192,10 @@ func TestCreateNoOp(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // skip role creation ig.GetEKSConfiguration().SetInstanceProfileName("some-profile") @@ -231,9 +237,10 @@ func TestCreateManagedRoleNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // Mock role/profile do not exist so they are always created @@ -280,9 +287,10 @@ func TestCreateLaunchConfigNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) iamMock.Role = &iam.Role{ @@ -314,9 +322,10 @@ func TestCreateAutoScalingGroupNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) iamMock.Role = &iam.Role{ @@ -332,3 +341,57 @@ func TestCreateAutoScalingGroupNegative(t *testing.T) { g.Expect(err).To(gomega.HaveOccurred()) g.Expect(ctx.GetState()).To(gomega.Equal(v1alpha1.ReconcileModifying)) } + +func TestCreateLatestAMI(t *testing.T) { + var ( + g = gomega.NewGomegaWithT(t) + k = MockKubernetesClientSet() + ig = MockInstanceGroup() + asgMock = NewAutoScalingMocker() + iamMock = NewIamMocker() + eksMock = NewEksMocker() + ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() + ) + + testLatestAmiID := "ami-12345678" + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) + ctx := MockContext(ig, k, w) + + // skip role creation + ig.GetEKSConfiguration().SetInstanceProfileName("some-profile") + ig.GetEKSConfiguration().SetRoleName("some-role") + iamMock.Role = &iam.Role{ + Arn: aws.String("some-arn"), + RoleName: aws.String("some-role"), + } + + // Setup Latest AMI + ig.GetEKSConfiguration().Image = "latest" + ssmMock.latestAMI = testLatestAmiID + + ec2Mock.InstanceTypes = []*ec2.InstanceTypeInfo{ + &ec2.InstanceTypeInfo{ + InstanceType: aws.String("m5.large"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String("x86_64")}, + }, + }, + } + + err := ctx.CloudDiscovery() + g.Expect(err).NotTo(gomega.HaveOccurred()) + // Must happen after ctx.CloudDiscover() + ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String("m5.large"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String("x86_64")}, + }, + }, + }) + + err = ctx.Create() + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(testLatestAmiID)) +} diff --git a/controllers/provisioners/eks/delete_test.go b/controllers/provisioners/eks/delete_test.go index d2277449..c591680d 100644 --- a/controllers/provisioners/eks/delete_test.go +++ b/controllers/provisioners/eks/delete_test.go @@ -41,9 +41,10 @@ func TestDeletePositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -72,9 +73,10 @@ func TestDeleteManagedRoleNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -108,9 +110,10 @@ func TestDeleteLaunchConfigurationNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -142,9 +145,10 @@ func TestDeleteAutoScalingGroupNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -174,9 +178,10 @@ func TestRemoveAuthRoleNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // two instancegroups with same role arn diff --git a/controllers/provisioners/eks/eks.go b/controllers/provisioners/eks/eks.go index bbd68b00..dd15a5c1 100644 --- a/controllers/provisioners/eks/eks.go +++ b/controllers/provisioners/eks/eks.go @@ -58,6 +58,7 @@ var ( DefaultManagedPolicies = []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"} CNIManagedPolicy = "AmazonEKS_CNI_Policy" AutoscalingReadOnlyPolicy = "AutoScalingReadOnlyAccess" + SupportedArchitectures = []string{"x86_64", "arm64"} ) // New constructs a new instance group provisioner of EKS type @@ -197,3 +198,7 @@ func (p *InstancePool) GetPool(key string) ([]InstanceSpec, bool) { } return nil, false } + +func (ctx *EksInstanceGroupContext) Locked() bool { + return ctx.InstanceGroup.Locked() +} diff --git a/controllers/provisioners/eks/eks_test.go b/controllers/provisioners/eks/eks_test.go index 47d693ad..a50c1c60 100644 --- a/controllers/provisioners/eks/eks_test.go +++ b/controllers/provisioners/eks/eks_test.go @@ -29,6 +29,8 @@ import ( "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/keikoproj/instance-manager/api/v1alpha1" "github.com/keikoproj/instance-manager/controllers/common" awsprovider "github.com/keikoproj/instance-manager/controllers/providers/aws" @@ -74,12 +76,17 @@ func NewEc2Mocker() *MockEc2Client { return &MockEc2Client{} } -func MockAwsWorker(asgClient *MockAutoScalingClient, iamClient *MockIamClient, eksClient *MockEksClient, ec2Client *MockEc2Client) awsprovider.AwsWorker { +func NewSsmMocker() *MockSsmClient { + return &MockSsmClient{} +} + +func MockAwsWorker(asgClient *MockAutoScalingClient, iamClient *MockIamClient, eksClient *MockEksClient, ec2Client *MockEc2Client, ssmClient *MockSsmClient) awsprovider.AwsWorker { return awsprovider.AwsWorker{ Ec2Client: ec2Client, AsgClient: asgClient, IamClient: iamClient, EksClient: eksClient, + SsmClient: ssmClient, } } @@ -839,3 +846,16 @@ func (i *MockIamClient) GetInstanceProfile(input *iam.GetInstanceProfileInput) ( func (i *MockIamClient) WaitUntilInstanceProfileExists(input *iam.GetInstanceProfileInput) error { return i.WaitUntilInstanceProfileExistsErr } + +type MockSsmClient struct { + ssmiface.SSMAPI + latestAMI string +} + +func (i *MockSsmClient) GetParameter(input *ssm.GetParameterInput) (*ssm.GetParameterOutput, error) { + return &ssm.GetParameterOutput{ + Parameter: &ssm.Parameter{ + Value: aws.String(i.latestAMI), + }, + }, nil +} diff --git a/controllers/provisioners/eks/helpers.go b/controllers/provisioners/eks/helpers.go index ea524e78..6ff3a3b7 100644 --- a/controllers/provisioners/eks/helpers.go +++ b/controllers/provisioners/eks/helpers.go @@ -505,13 +505,13 @@ func (ctx *EksInstanceGroupContext) GetComputedBootstrapOptions() *v1alpha1.Boot var prefixAssignmentEnabled = instanceGroup.GetAnnotations()[CustomNetworkingPrefixAssignmentEnabledAnnotation] == "true" var maxPods int64 = 0 - var enis = aws.Int64Value(instanceTypeNetworkInfo.MaximumNetworkInterfaces)-1 //Primary interface is not used for pod networking when custom networking is enabled + var enis = aws.Int64Value(instanceTypeNetworkInfo.MaximumNetworkInterfaces) - 1 //Primary interface is not used for pod networking when custom networking is enabled var ipsPerInterface int64 = 1 if prefixAssignmentEnabled { ipsPerInterface = 16 //Number of ips in a /28 block } - maxPods = enis * ((aws.Int64Value(instanceTypeNetworkInfo.Ipv4AddressesPerInterface)-1) * ipsPerInterface)+ hostNetworkPods + maxPods = enis*((aws.Int64Value(instanceTypeNetworkInfo.Ipv4AddressesPerInterface)-1)*ipsPerInterface) + hostNetworkPods if configuration.BootstrapOptions == nil { return &v1alpha1.BootstrapOptions{ @@ -1173,3 +1173,40 @@ func (ctx *EksInstanceGroupContext) GetDesiredMixedInstancesPolicy(name string) return policy } + +func FilterSupportedArch(architectures []string) string { + for _, a := range architectures { + for _, supportedArch := range SupportedArchitectures { + result := a == supportedArch + if result { + return supportedArch + } + } + } + return "" +} + +func (ctx *EksInstanceGroupContext) GetEksLatestAmi() (string, error) { + var ( + instanceGroup = ctx.GetInstanceGroup() + state = ctx.GetDiscoveredState() + configuration = instanceGroup.GetEKSConfiguration() + ) + clusterVersion := state.GetClusterVersion() + annotations := instanceGroup.GetAnnotations() + + var OSFamily string + if kubeprovider.HasAnnotation(annotations, OsFamilyAnnotation) { + OSFamily = annotations[OsFamilyAnnotation] + } else { + OSFamily = OsFamilyAmazonLinux2 + } + + supportedArchitectures := awsprovider.GetInstanceTypeArchitectures(state.GetInstanceTypeInfo(), configuration.InstanceType) + arch := FilterSupportedArch(supportedArchitectures) + if arch == "" { + return "", fmt.Errorf("No supported CPU architecture found for instance type %s", configuration.InstanceType) + } + + return ctx.AwsWorker.GetEksLatestAmi(OSFamily, arch, clusterVersion) +} diff --git a/controllers/provisioners/eks/helpers_test.go b/controllers/provisioners/eks/helpers_test.go index ec2d826e..9e09ff44 100644 --- a/controllers/provisioners/eks/helpers_test.go +++ b/controllers/provisioners/eks/helpers_test.go @@ -42,9 +42,10 @@ func TestAutoscalerTags(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ig.Annotations = map[string]string{ ClusterAutoscalerEnabledAnnotation: "true", @@ -89,10 +90,11 @@ func TestGetBasicUserDataAmazonLinux2(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() configuration = ig.GetEKSConfiguration() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{ @@ -179,10 +181,11 @@ func TestGetBasicUserDataWindows(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() configuration = ig.GetEKSConfiguration() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) configuration.BootstrapOptions = &v1alpha1.BootstrapOptions{ @@ -254,9 +257,10 @@ func TestCustomNetworkingMaxPods(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ig.Annotations = map[string]string{ ClusterAutoscalerEnabledAnnotation: "true", @@ -316,10 +320,10 @@ func TestCustomNetworkingMaxPods(t *testing.T) { }, { annotations: map[string]string{ - ClusterAutoscalerEnabledAnnotation: "true", + ClusterAutoscalerEnabledAnnotation: "true", CustomNetworkingPrefixAssignmentEnabledAnnotation: "true", - CustomNetworkingHostPodsAnnotation: "2", - CustomNetworkingEnabledAnnotation: "true", + CustomNetworkingHostPodsAnnotation: "2", + CustomNetworkingEnabledAnnotation: "true", }, bootstrapOptions: nil, expectedMaxPods: "--max-pods=290", @@ -357,9 +361,10 @@ func TestResolveSecurityGroups(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -399,9 +404,10 @@ func TestResolveSubnets(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -440,9 +446,10 @@ func TestGetDisabledMetrics(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // No disable required @@ -500,9 +507,10 @@ func TestGetEnabledMetrics(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // Enable all metrics @@ -547,6 +555,7 @@ func TestGetLabelList(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() defaultLifecycleLabel = "instancemgr.keikoproj.io/lifecycle=normal" defaultImageLabel = fmt.Sprintf("instancemgr.keikoproj.io/image=%v", configuration.GetImage()) expectedLabels115 = []string{defaultImageLabel, defaultLifecycleLabel, "node-role.kubernetes.io/instance-group-1=\"\"", "node.kubernetes.io/role=instance-group-1"} @@ -558,7 +567,7 @@ func TestGetLabelList(t *testing.T) { expectedMixedLabel = []string{defaultImageLabel, "instancemgr.keikoproj.io/lifecycle=mixed", "node-role.kubernetes.io/instance-group-1=\"\"", "node.kubernetes.io/role=instance-group-1"} ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -615,9 +624,10 @@ func TestGetMountOpts(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) volumeNoOpts := v1alpha1.NodeVolume{ @@ -728,9 +738,10 @@ func TestGetUserDataStages(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -785,9 +796,10 @@ func TestMaxPodsSetCorrectly(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) bottleRocketIgWithMaxPods.Spec.EKSSpec.EKSConfiguration.BootstrapOptions = &v1alpha1.BootstrapOptions{ MaxPods: 15, @@ -843,9 +855,10 @@ func TestBootstrapDataForOSFamily(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) tests := []struct { ig *v1alpha1.InstanceGroup @@ -888,9 +901,10 @@ func TestUpdateLifecycleHooks(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) testScalingHooks := []*autoscaling.LifecycleHook{ @@ -965,9 +979,10 @@ func TestUpdateWarmPool(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -1022,3 +1037,102 @@ func TestUpdateWarmPool(t *testing.T) { } } } + +func TestFilterSupportedArch(t *testing.T) { + var ( + g = gomega.NewGomegaWithT(t) + ) + + tests := []struct { + name string + architectures []string + expected string + }{ + { + name: "supported architecture x86", + architectures: []string{"x86_64"}, + expected: "x86_64", + }, + { + name: "supported architecture arm64", + architectures: []string{"arm64"}, + expected: "arm64", + }, + { + name: "no supported architecture", + architectures: []string{}, + expected: "", + }, + } + + for _, tc := range tests { + result := FilterSupportedArch(tc.architectures) + g.Expect(result).To(gomega.Equal(tc.expected)) + } + +} + +func TestGetEksLatestAmi(t *testing.T) { + var ( + k = MockKubernetesClientSet() + ig = MockInstanceGroup() + config = ig.GetEKSConfiguration() + asgMock = NewAutoScalingMocker() + iamMock = NewIamMocker() + eksMock = NewEksMocker() + ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() + instanceType = "m5.large" + ) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) + + tests := []struct { + name string + OSFamily string + arch string + expectedError error + }{ + { + name: "AmazonLinux2-x86_64", + OSFamily: "amazonlinux2", + arch: "x86_64", + expectedError: nil, + }, + { + name: "bottlerocket-x86_64", + OSFamily: "bottlerocket", + arch: "x86_64", + expectedError: nil, + }, + { + name: "AmazonLinux2-noarch", + OSFamily: "amazonlinux2", + arch: "noarch", + expectedError: fmt.Errorf("No supported CPU architecture found for instance type %s", instanceType), + }, + } + + for _, tc := range tests { + ig.SetAnnotations(map[string]string{ + OsFamilyAnnotation: tc.OSFamily, + }) + config.InstanceType = instanceType + ctx := MockContext(ig, k, w) + ctx.GetDiscoveredState().SetInstanceTypeInfo([]*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String(instanceType), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String(tc.arch)}, + }, + }, + }) + _, err := ctx.GetEksLatestAmi() + if err == nil && tc.expectedError == nil { + continue + } + if err != nil && tc.expectedError != nil && err.Error() != tc.expectedError.Error() { + t.Fatalf("expected %v got %v, test %s", tc.expectedError, err, tc.name) + } + + } +} diff --git a/controllers/provisioners/eks/state_test.go b/controllers/provisioners/eks/state_test.go index dcb4a01f..7d6f86da 100644 --- a/controllers/provisioners/eks/state_test.go +++ b/controllers/provisioners/eks/state_test.go @@ -36,9 +36,10 @@ func TestStateDiscovery(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -87,9 +88,10 @@ func TestIsReady(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { diff --git a/controllers/provisioners/eks/update_test.go b/controllers/provisioners/eks/update_test.go index f676174b..3be098c8 100644 --- a/controllers/provisioners/eks/update_test.go +++ b/controllers/provisioners/eks/update_test.go @@ -45,9 +45,10 @@ func TestUpdateWithDriftRotationPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) mockScalingGroup := &autoscaling.Group{ @@ -131,9 +132,10 @@ func TestUpdateWithLaunchTemplate(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) spec.Type = v1alpha1.LaunchTemplate @@ -304,9 +306,10 @@ func TestUpdateWithRotationPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -381,9 +384,10 @@ func TestLaunchConfigurationDrifted(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ @@ -488,9 +492,10 @@ func TestUpdateScalingGroupNegative(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ig.GetEKSConfiguration().SetMetricsCollection([]string{"GroupMinSize", "GroupMaxSize", "GroupDesiredCapacity"}) @@ -563,9 +568,10 @@ func TestScalingGroupUpdatePredicate(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) spec.MinSize = int64(3) spec.MaxSize = int64(6) @@ -621,13 +627,14 @@ func TestUpdateManagedPolicies(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) defaultPolicies := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly"} defaultPoliciesIrsa := []string{"AmazonEKSWorkerNodePolicy", "AmazonEC2ContainerRegistryReadOnly"} defaultPoliciesWarmPool := []string{"AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly", "AutoScalingReadOnlyAccess"} - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -693,3 +700,83 @@ func TestUpdateManagedPolicies(t *testing.T) { g.Expect(iamMock.DetachRolePolicyCallCount).To(gomega.Equal(tc.expectedDetached)) } } + +func TestUpdateWithLatestAmiID(t *testing.T) { + var ( + g = gomega.NewGomegaWithT(t) + k = MockKubernetesClientSet() + ig = MockInstanceGroup() + asgMock = NewAutoScalingMocker() + iamMock = NewIamMocker() + eksMock = NewEksMocker() + ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() + ) + + testLatestAmiID := "ami-12345678" + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) + ctx := MockContext(ig, k, w) + + mockScalingGroup := &autoscaling.Group{ + AutoScalingGroupName: aws.String("some-scaling-group"), + DesiredCapacity: aws.Int64(1), + Instances: []*autoscaling.Instance{ + { + InstanceId: aws.String("i-1234"), + LaunchConfigurationName: aws.String("some-launch-config"), + }, + }, + } + asgMock.AutoScalingGroups = []*autoscaling.Group{mockScalingGroup} + + // skip role creation + ig.GetEKSConfiguration().SetInstanceProfileName("some-profile") + ig.GetEKSConfiguration().SetRoleName("some-role") + iamMock.Role = &iam.Role{ + Arn: aws.String("some-arn"), + RoleName: aws.String("some-role"), + } + + // Setup Latest AMI + ig.GetEKSConfiguration().Image = "latest" + ssmMock.latestAMI = testLatestAmiID + + ec2Mock.InstanceTypes = []*ec2.InstanceTypeInfo{ + &ec2.InstanceTypeInfo{ + InstanceType: aws.String("m5.large"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String("x86_64")}, + }, + }, + } + + err := ctx.CloudDiscovery() + g.Expect(err).NotTo(gomega.HaveOccurred()) + + ctx.SetDiscoveredState(&DiscoveredState{ + Publisher: kubeprovider.EventPublisher{ + Client: k.Kubernetes, + }, + ScalingGroup: mockScalingGroup, + ScalingConfiguration: &scaling.LaunchConfiguration{ + AwsWorker: w, + }, + InstanceProfile: &iam.InstanceProfile{ + Arn: aws.String("some-instance-arn"), + }, + ClusterNodes: nil, + Cluster: nil, + InstanceTypeInfo: []*ec2.InstanceTypeInfo{ + { + InstanceType: aws.String("m5.large"), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: []*string{aws.String("x86_64")}, + }, + }, + }, + }) + + err = ctx.Update() + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(ctx.GetInstanceGroup().Spec.EKSSpec.EKSConfiguration.Image).To(gomega.Equal(testLatestAmiID)) +} diff --git a/controllers/provisioners/eks/upgrade_test.go b/controllers/provisioners/eks/upgrade_test.go index 660ca3a7..12bb7362 100644 --- a/controllers/provisioners/eks/upgrade_test.go +++ b/controllers/provisioners/eks/upgrade_test.go @@ -115,9 +115,10 @@ func TestUpgradeInvalidStrategy(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // assume initial state of modifying @@ -138,9 +139,10 @@ func TestBootstrapNodes(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) // assume initial state of modifying @@ -160,9 +162,10 @@ func TestUpgradeCRDStrategy(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) ctx.SetDiscoveredState(&DiscoveredState{ Publisher: kubeprovider.EventPublisher{ @@ -231,9 +234,10 @@ func TestUpgradeRollingUpdateStrategyPositive(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { @@ -341,9 +345,10 @@ func TestRotateWarmPool(t *testing.T) { iamMock = NewIamMocker() eksMock = NewEksMocker() ec2Mock = NewEc2Mocker() + ssmMock = NewSsmMocker() ) - w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock) + w := MockAwsWorker(asgMock, iamMock, eksMock, ec2Mock, ssmMock) ctx := MockContext(ig, k, w) tests := []struct { diff --git a/controllers/provisioners/eksfargate/eksfargate.go b/controllers/provisioners/eksfargate/eksfargate.go index 5843a71c..cad8d5a6 100644 --- a/controllers/provisioners/eksfargate/eksfargate.go +++ b/controllers/provisioners/eksfargate/eksfargate.go @@ -366,3 +366,6 @@ func (ctx *FargateInstanceGroupContext) SetState(state v1alpha1.ReconcileState) func (ctx *FargateInstanceGroupContext) GetState() v1alpha1.ReconcileState { return ctx.GetInstanceGroup().GetState() } +func (ctx *FargateInstanceGroupContext) Locked() bool { + return false +} diff --git a/controllers/provisioners/eksmanaged/eksmanaged.go b/controllers/provisioners/eksmanaged/eksmanaged.go index 63e23599..fcf7af4b 100644 --- a/controllers/provisioners/eksmanaged/eksmanaged.go +++ b/controllers/provisioners/eksmanaged/eksmanaged.go @@ -215,6 +215,10 @@ func (ctx *EksManagedInstanceGroupContext) IsReady() bool { return false } +func (ctx *EksManagedInstanceGroupContext) Locked() bool { + return false +} + func New(p provisioners.ProvisionerInput) *EksManagedInstanceGroupContext { ctx := &EksManagedInstanceGroupContext{ diff --git a/controllers/provisioners/provisioners.go b/controllers/provisioners/provisioners.go index 48a92529..b2985ee7 100644 --- a/controllers/provisioners/provisioners.go +++ b/controllers/provisioners/provisioners.go @@ -23,6 +23,7 @@ const ( TagKubernetesCluster = "KubernetesCluster" ConfigurationExclusionAnnotationKey = "instancemgr.keikoproj.io/config-excluded" + UpgradeLockedAnnotationKey = "instancemgr.keikoproj.io/lock-upgrades" ) type ProvisionerInput struct { @@ -36,7 +37,7 @@ type ProvisionerInput struct { } var ( - NonRetryableStates = []v1alpha1.ReconcileState{v1alpha1.ReconcileErr, v1alpha1.ReconcileReady, v1alpha1.ReconcileDeleted} + NonRetryableStates = []v1alpha1.ReconcileState{v1alpha1.ReconcileErr, v1alpha1.ReconcileReady, v1alpha1.ReconcileDeleted, v1alpha1.ReconcileLocked} ) func IsRetryable(instanceGroup *v1alpha1.InstanceGroup) bool { diff --git a/docs/EKS.md b/docs/EKS.md index 9fac3ce2..2e4dfec4 100644 --- a/docs/EKS.md +++ b/docs/EKS.md @@ -657,3 +657,4 @@ The following operators are supported: |instancemgr.keikoproj.io/custom-networking-enabled|InstanceGroup|"true"|setting this annotation to true will automatically calculate the correct setting for max pods and pass it to the kubelet| |instancemgr.keikoproj.io/custom-networking-prefix-assignment-enabled|InstanceGroup|"true"|setting this annotation to true will change the max pod calculations to reflect the pod density supported by vpc prefix assignment. Supported in AWS VPC CNI versions 1.9.0 and above - see [AWS VPC CNI 1.9.0](https://github.com/aws/amazon-vpc-cni-k8s/releases/tag/v1.9.0) for more information.| |instancemgr.keikoproj.io/custom-networking-host-pods|InstanceGroup|"2"|setting this annotation increases the number of max pods on nodes with custom networking, due to the fact that hostNetwork pods do not use an additional IP address | +|instancemgr.keikoproj.io/lock-upgrades|InstanceGroup|bool|setting this annotation to true will prevent instance-manager from triggering upgrades to the nodes within an instance group. This is useful for controlling when an upgrade happens. Changes to this annotation will trigger a reconcile loop| diff --git a/docs/INSTALL.md b/docs/INSTALL.md index fefa9afb..7bbbc328 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -88,6 +88,7 @@ eks:DescribeNodegroup eks:DeleteNodegroup eks:UpdateNodegroupConfig eks:DescribeCluster +ssm:GetParameter ``` The following IAM permissions are required if you want the controller to be creating IAM roles for your instance groups, otherwise you can omit this and provide an existing role in the custom resource. diff --git a/docs/examples/EKS.md b/docs/examples/EKS.md index 90baf866..5c14efb7 100644 --- a/docs/examples/EKS.md +++ b/docs/examples/EKS.md @@ -7,6 +7,7 @@ - [Upgrade Strategies](#upgrade-strategies) - [EC2 Instance placement](#ec2-instance-placement) - [Bring your own role](#bring-your-own-role) + - [Auto-Updating AWS EKS Optimised AMI ID](#auto-updating-aws-eks-optimised-ami-id) ## EKS sample spec @@ -177,3 +178,28 @@ spec: ``` if you do not provide these fields, a role will be created for your instance-group by the controller (will require IAM access). + + +### Auto-Updating AWS EKS Optimised AMI ID + +Amazon provides a EKS optimised AMIs for AmazonLinux2, Bottlerocket and Windows - https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html +Instance-manager can periodically query the latest AMI ID from AWS SSM parameter store, and automatically update the image used in your instance groups, ensuring your nodes are always patched and kept up-to-date. This feature pairs well with customised upgrade strategies to ensure nodes are safely upgraded when and how you expect. + +note: The controller reconciles `InstanceGroups` every 10 hours by default, so changes may take up to 11 hours (sync period + SSM GetParameter cache TTL) to occur. +Using the locking annotation `instancemgr.keikoproj.io/lock-upgrades` will allow more control of upgrades. Locking/Unlocking `InstanceGroups` will trigger a reconcile. + +```yaml +apiVersion: instancemgr.keikoproj.io/v1alpha1 +kind: InstanceGroup +metadata: + name: hello-world + namespace: instance-manager + annotations: + instancemgr.keikoproj.io/os-family: "amazonlinux2" #Default if not provided +spec: + strategy: <...> + provisioner: eks + eks: + configuration: + image: latest +``` \ No newline at end of file diff --git a/main.go b/main.go index caaa63bc..fbc4151c 100644 --- a/main.go +++ b/main.go @@ -125,6 +125,7 @@ func main() { IamClient: aws.GetAwsIamClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector), AsgClient: aws.GetAwsAsgClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector), EksClient: aws.GetAwsEksClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector), + SsmClient: aws.GetAwsSsmClient(awsRegion, cacheCfg, maxAPIRetries, controllerCollector), Ec2Metadata: metadata, } diff --git a/test-bdd/features/01_create.feature b/test-bdd/features/01_create.feature index 2a9f84e5..34ab3214 100644 --- a/test-bdd/features/01_create.feature +++ b/test-bdd/features/01_create.feature @@ -17,6 +17,7 @@ Feature: CRUD Create And I create a resource instance-group-launch-template-mixed.yaml And I create a resource manager-configmap.yaml And I create a resource instance-group-gitops.yaml + And I create a resource instance-group-latest-locked.yaml Scenario: Create an instance-group with rollingUpdate strategy Given an EKS cluster @@ -88,3 +89,11 @@ Feature: CRUD Create And the resource should converge to selector .status.currentState=ready And the resource condition NodesReady should be true And 2 nodes should be ready + + Scenario: Create an instance-group with latest ami + Given an EKS cluster + When I create a resource instance-group-latest-locked.yaml + Then the resource should be created + And the resource should converge to selector .status.currentState=ready + And the resource condition NodesReady should be true + And 2 nodes should be ready diff --git a/test-bdd/features/02_update.feature b/test-bdd/features/02_update.feature index 837b7f81..0c13d72a 100644 --- a/test-bdd/features/02_update.feature +++ b/test-bdd/features/02_update.feature @@ -12,6 +12,7 @@ Feature: CRUD Update And I update a resource instance-group-launch-template.yaml with .spec.eks.minSize set to 3 And I update a resource instance-group-launch-template-mixed.yaml with .spec.eks.minSize set to 3 And I update a resource instance-group-managed.yaml with .spec.eks-managed.minSize set to 3 + And I update a resource instance-group-latest-locked.yaml with .spec.eks.minSize set to 3 Scenario: Update an instance-group with rollingUpdate strategy Given an EKS cluster @@ -60,3 +61,9 @@ Feature: CRUD Update When I update a resource instance-group-managed.yaml with .spec.eks-managed.minSize set to 3 Then the resource should converge to selector .status.currentState=ready And 3 nodes should be ready + + Scenario: Update an instance-group with latest ami + Given an EKS cluster + When I update a resource instance-group-latest-locked.yaml with .spec.eks.minSize set to 3 + Then the resource should converge to selector .status.currentState=ready + And 3 nodes should be ready \ No newline at end of file diff --git a/test-bdd/features/03_upgrade.feature b/test-bdd/features/03_upgrade.feature index 2209b66e..30a9a704 100644 --- a/test-bdd/features/03_upgrade.feature +++ b/test-bdd/features/03_upgrade.feature @@ -61,3 +61,13 @@ Feature: CRUD Upgrade And the resource should converge to selector .status.currentState=ready And the resource condition NodesReady should be true And 3 nodes should be ready + + Scenario: Lock an instance-group + Given an EKS cluster + When I update a resource instance-group-latest-locked.yaml with annotation instancemgr.keikoproj.io/lock-upgrades set to true + Then I update a resource instance-group-latest-locked.yaml with .spec.eks.configuration.instanceType set to t2.medium + And the resource should converge to selector .status.currentState=locked + And I update a resource instance-group-latest-locked.yaml with annotation instancemgr.keikoproj.io/lock-upgrades set to false + And the resource should converge to selector .status.currentState=ready + And the resource condition NodesReady should be true + And 3 nodes should be ready diff --git a/test-bdd/features/04_delete.feature b/test-bdd/features/04_delete.feature index c9b50a48..57bacbc3 100644 --- a/test-bdd/features/04_delete.feature +++ b/test-bdd/features/04_delete.feature @@ -14,6 +14,7 @@ Feature: CRUD Delete And I delete a resource instance-group-managed.yaml And I delete a resource instance-group-fargate.yaml And I delete a resource instance-group-gitops.yaml + And I delete a resource instance-group-latest-locked.yaml Scenario: Delete an instance-group with rollingUpdate strategy Given an EKS cluster @@ -49,6 +50,12 @@ Feature: CRUD Delete Given an EKS cluster Then I delete a resource instance-group-fargate.yaml And the resource should be deleted + + Scenario: Delete a locked profile + Given an EKS cluster + When I delete a resource instance-group-latest-locked.yaml + Then 0 nodes should be found + And the resource should be deleted Scenario: Delete an instance-group with shortened resource Given an EKS cluster diff --git a/test-bdd/main_test.go b/test-bdd/main_test.go index 1260520d..9bf716ff 100644 --- a/test-bdd/main_test.go +++ b/test-bdd/main_test.go @@ -114,6 +114,7 @@ func FeatureContext(s *godog.Suite) { time.Sleep(time.Second * 5) }) + // Order matters s.Step(`^an EKS cluster`, t.anEKSCluster) s.Step(`^(\d+) nodes should be (found|ready)`, t.nodesShouldBe) s.Step(`^(\d+) nodes should be (found|ready) with label ([^"]*) set to ([^"]*)$`, t.nodesShouldBeWithLabel) @@ -121,6 +122,7 @@ func FeatureContext(s *godog.Suite) { s.Step(`^the resource should converge to selector ([^"]*)$`, t.theResourceShouldConvergeToSelector) s.Step(`^the resource condition ([^"]*) should be (true|false)$`, t.theResourceConditionShouldBe) s.Step(`^I (create|delete) a resource ([^"]*)$`, t.iOperateOnResource) + s.Step(`^I update a resource ([^"]*) with annotation ([^"]*) set to ([^"]*)$`, t.iUpdateResourceWithAnnotation) s.Step(`^I update a resource ([^"]*) with ([^"]*) set to ([^"]*)$`, t.iUpdateResourceWithField) } @@ -205,6 +207,34 @@ func (t *FunctionalTest) iOperateOnResource(operation, fileName string) error { return nil } +func (t *FunctionalTest) iUpdateResourceWithAnnotation(fileName, annotation string, value string) error { + resourcePath := filepath.Join("templates", fileName) + args := testutil.NewTemplateArguments() + + gvr, resource, err := testutil.GetResourceFromYaml(resourcePath, t.RESTConfig, args) + if err != nil { + return err + } + + t.ResourceName = resource.GetName() + t.ResourceNamespace = resource.GetNamespace() + + updateTarget, err := t.DynamicClient.Resource(gvr.Resource).Namespace(t.ResourceNamespace).Get(context.Background(), t.ResourceName, metav1.GetOptions{}) + if err != nil { + return err + } + + unstructured.SetNestedField(updateTarget.UnstructuredContent(), value, []string{"metadata", "annotations", annotation}...) + + _, err = t.DynamicClient.Resource(InstanceGroupSchema).Namespace(t.ResourceNamespace).Update(context.Background(), updateTarget, metav1.UpdateOptions{}) + if err != nil { + return err + } + time.Sleep(3 * time.Second) + return nil + +} + func (t *FunctionalTest) iUpdateResourceWithField(fileName, key string, value string) error { var ( keySlice = testutil.DeleteEmpty(strings.Split(key, "."))