Skip to content

Commit

Permalink
Tag Check Changes
Browse files Browse the repository at this point in the history
  • Loading branch information
edibble21 committed Nov 24, 2024
1 parent 317f551 commit d2ca00b
Show file tree
Hide file tree
Showing 11 changed files with 146 additions and 9 deletions.
1 change: 1 addition & 0 deletions cmd/controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ func main() {
ctx,
op.Manager,
op.Config,
op.EC2API,
op.Clock,
op.GetClient(),
op.EventRecorder,
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/v1/ec2nodeclass_status.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ const (
ConditionTypeSecurityGroupsReady = "SecurityGroupsReady"
ConditionTypeAMIsReady = "AMIsReady"
ConditionTypeInstanceProfileReady = "InstanceProfileReady"
ConditionTypeTagsReady = "TagsReady"
)

// Subnet contains resolved Subnet selector values utilized for node launch
Expand Down Expand Up @@ -90,6 +91,7 @@ func (in *EC2NodeClass) StatusConditions() status.ConditionSet {
ConditionTypeSubnetsReady,
ConditionTypeSecurityGroupsReady,
ConditionTypeInstanceProfileReady,
ConditionTypeTagsReady,
).For(in)
}

Expand Down
6 changes: 3 additions & 3 deletions pkg/cloudprovider/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1148,7 +1148,7 @@ var _ = Describe("CloudProvider", func() {
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(100),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.EC2API)
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
ExpectObjectReconciled(ctx, env.Client, controller, nodeClass)
pod := coretest.UnschedulablePod(coretest.PodOptions{NodeSelector: map[string]string{corev1.LabelTopologyZone: "test-zone-1a"}})
Expand All @@ -1165,7 +1165,7 @@ var _ = Describe("CloudProvider", func() {
{SubnetId: aws.String("test-subnet-2"), AvailabilityZone: aws.String("test-zone-1a"), AvailabilityZoneId: aws.String("tstz1-1a"), AvailableIpAddressCount: aws.Int32(11),
Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String("test-subnet-2")}}},
}})
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.EC2API)
nodeClass.Spec.Kubelet = &v1.KubeletConfiguration{
MaxPods: aws.Int32(1),
}
Expand Down Expand Up @@ -1206,7 +1206,7 @@ var _ = Describe("CloudProvider", func() {
}})
nodeClass.Spec.SubnetSelectorTerms = []v1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-1"}}}
ExpectApplied(ctx, env.Client, nodePool, nodeClass)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.EC2API)
ExpectObjectReconciled(ctx, env.Client, controller, nodeClass)
podSubnet1 := coretest.UnschedulablePod()
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, podSubnet1)
Expand Down
4 changes: 3 additions & 1 deletion pkg/controllers/controllers.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
nodeclasshash "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/hash"
nodeclassstatus "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status"
nodeclasstermination "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/termination"
Expand Down Expand Up @@ -63,6 +64,7 @@ func NewControllers(
ctx context.Context,
mgr manager.Manager,
cfg aws.Config,
ec2api sdk.EC2API,
clk clock.Clock,
kubeClient client.Client,
recorder events.Recorder,
Expand All @@ -79,7 +81,7 @@ func NewControllers(
instanceTypeProvider *instancetype.DefaultProvider) []controller.Controller {
controllers := []controller.Controller{
nodeclasshash.NewController(kubeClient),
nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider),
nodeclassstatus.NewController(kubeClient, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider, ec2api),
nodeclasstermination.NewController(kubeClient, recorder, instanceProfileProvider, launchTemplateProvider),
nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider),
nodeclaimtagging.NewController(kubeClient, instanceProvider),
Expand Down
6 changes: 5 additions & 1 deletion pkg/controllers/nodeclass/status/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/awslabs/operatorpkg/reasonable"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
"github.com/aws/karpenter-provider-aws/pkg/providers/amifamily"
"github.com/aws/karpenter-provider-aws/pkg/providers/instanceprofile"
"github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate"
Expand All @@ -52,10 +53,11 @@ type Controller struct {
subnet *Subnet
securitygroup *SecurityGroup
readiness *Readiness //TODO : Remove this when we have sub status conditions
tags *Tags
}

func NewController(kubeClient client.Client, subnetProvider subnet.Provider, securityGroupProvider securitygroup.Provider,
amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider) *Controller {
amiProvider amifamily.Provider, instanceProfileProvider instanceprofile.Provider, launchTemplateProvider launchtemplate.Provider, ec2API sdk.EC2API) *Controller {
return &Controller{
kubeClient: kubeClient,

Expand All @@ -64,6 +66,7 @@ func NewController(kubeClient client.Client, subnetProvider subnet.Provider, sec
securitygroup: &SecurityGroup{securityGroupProvider: securityGroupProvider},
instanceprofile: &InstanceProfile{instanceProfileProvider: instanceProfileProvider},
readiness: &Readiness{launchTemplateProvider: launchTemplateProvider},
tags: &Tags{ec2API: ec2API},
}
}

Expand Down Expand Up @@ -94,6 +97,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass)
c.securitygroup,
c.instanceprofile,
c.readiness,
c.tags,
} {
res, err := reconciler.Reconcile(ctx, nodeClass)
errs = multierr.Append(errs, err)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controllers/nodeclass/status/readiness_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ var _ = Describe("NodeClass Status Condition Controller", func() {
ExpectApplied(ctx, env.Client, nodeClass)
ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
nodeClass = ExpectExists(ctx, env.Client, nodeClass)
Expect(nodeClass.Status.Conditions).To(HaveLen(5))
Expect(nodeClass.Status.Conditions).To(HaveLen(6))
Expect(nodeClass.StatusConditions().Get(status.ConditionReady).IsTrue()).To(BeTrue())
})
It("should update status condition as Not Ready", func() {
Expand Down
2 changes: 2 additions & 0 deletions pkg/controllers/nodeclass/status/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ var _ = BeforeSuite(func() {
awsEnv.AMIProvider,
awsEnv.InstanceProfileProvider,
awsEnv.LaunchTemplateProvider,
awsEnv.EC2API,
)
})

var _ = AfterSuite(func() {
awsEnv.Reset()
Expect(env.Stop()).To(Succeed(), "Failed to stop environment")
})

Expand Down
58 changes: 58 additions & 0 deletions pkg/controllers/nodeclass/status/tags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package status

import (
"context"
"errors"
"fmt"
"time"

"sigs.k8s.io/controller-runtime/pkg/reconcile"

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
"github.com/aws/smithy-go"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
sdk "github.com/aws/karpenter-provider-aws/pkg/aws"
"github.com/aws/karpenter-provider-aws/pkg/utils"
)

type Tags struct {
ec2API sdk.EC2API
}

func (ip *Tags) Reconcile(ctx context.Context, nodeClass *v1.EC2NodeClass) (reconcile.Result, error) {
if nodeClass.Spec.Role != "" {
createLaunchTemplateInput := &ec2.CreateLaunchTemplateInput{
DryRun: aws.Bool(true),
TagSpecifications: []ec2types.TagSpecification{
{ResourceType: ec2types.ResourceTypeInstance, Tags: utils.MergeTags(nodeClass.Spec.Tags)},
{ResourceType: ec2types.ResourceTypeVolume, Tags: utils.MergeTags(nodeClass.Spec.Tags)},
{ResourceType: ec2types.ResourceTypeFleet, Tags: utils.MergeTags(nodeClass.Spec.Tags)},
},
}
_, err := ip.ec2API.CreateLaunchTemplate(ctx, createLaunchTemplateInput)
var APIErr smithy.APIError
if err != nil && errors.As(err, &APIErr) && APIErr.ErrorCode() == "UnauthorizedOperation" {
nodeClass.StatusConditions().SetFalse(v1.ConditionTypeTagsReady, "UnauthorizedOperation", fmt.Sprintf("role does not contain required permissions: %v", err.Error()))
return reconcile.Result{}, fmt.Errorf("reconciling Tags, %w", err)
}
}
nodeClass.StatusConditions().SetTrue(v1.ConditionTypeTagsReady)
return reconcile.Result{RequeueAfter: time.Minute}, nil
}
62 changes: 62 additions & 0 deletions pkg/controllers/nodeclass/status/tags_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package status_test

import (
"github.com/aws/smithy-go"

v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
"github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclass/status"
"github.com/aws/karpenter-provider-aws/pkg/fake"
"github.com/aws/karpenter-provider-aws/pkg/test"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "sigs.k8s.io/karpenter/pkg/test/expectations"
)

var _ = Describe("NodeClass Tags Status Controller", func() {
BeforeEach(func() {
nodeClass = test.EC2NodeClass(v1.EC2NodeClass{
Spec: v1.EC2NodeClassSpec{
Tags: map[string]string{
"fakeKey": "fakeValue",
},
},
})
})
BeforeEach(func() {
awsEnv.Reset()
})
It("Should update EC2NodeClass status for Tags", func() {
ExpectApplied(ctx, env.Client, nodeClass)
ExpectObjectReconciled(ctx, env.Client, statusController, nodeClass)
nodeClass = ExpectExists(ctx, env.Client, nodeClass)
Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeTagsReady)).To(BeTrue())
})
It("Should not update EC2NodeClass status for Tags without appropriate permissions", func() {
ec2api := fake.NewEC2API()
ec2api.NextError.Set(&smithy.GenericAPIError{
Code: "UnauthorizedOperation",
})
ExpectApplied(ctx, env.Client, nodeClass)
statusController := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, ec2api)
_, err := statusController.Reconcile(ctx, nodeClass)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("UnauthorizedOperation"))
nodeClass = ExpectExists(ctx, env.Client, nodeClass)
Expect(nodeClass.StatusConditions().IsTrue(v1.ConditionTypeTagsReady)).To(BeFalse())
})
})
2 changes: 2 additions & 0 deletions pkg/operator/operator.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func init() {
type Operator struct {
*operator.Operator
Config aws.Config
EC2API sdk.EC2API
UnavailableOfferingsCache *awscache.UnavailableOfferings
SSMCache *cache.Cache
SubnetProvider subnet.Provider
Expand Down Expand Up @@ -163,6 +164,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
return ctx, &Operator{
Operator: operator,
Config: cfg,
EC2API: ec2api,
UnavailableOfferingsCache: unavailableOfferingsCache,
SSMCache: ssmCache,
SubnetProvider: subnetProvider,
Expand Down
10 changes: 7 additions & 3 deletions pkg/providers/launchtemplate/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2025,7 +2025,7 @@ essential = true
nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyCustom)
nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}}
ExpectApplied(ctx, env.Client, nodeClass)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider)
controller := status.NewController(env.Client, awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider, awsEnv.EC2API)
ExpectObjectReconciled(ctx, env.Client, controller, nodeClass)
nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{
{
Expand All @@ -2040,10 +2040,14 @@ essential = true
pod := coretest.UnschedulablePod()
ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod)
ExpectScheduled(ctx, env.Client, pod)
Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically("==", 1))
Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically("==", 2)) // once for the reconciler and once for launch template creation
containedAmi := 0
awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) {
Expect("ami-456").To(Equal(*ltInput.LaunchTemplateData.ImageId))
if ltInput.LaunchTemplateData != nil && ltInput.LaunchTemplateData.ImageId != nil && *ltInput.LaunchTemplateData.ImageId == "ami-456" {
containedAmi++
}
})
Expect(containedAmi).To(Equal(1))
})

It("should fail if no amis match selector.", func() {
Expand Down

0 comments on commit d2ca00b

Please sign in to comment.