From ba990907ecf1059ad35e16e973877cd3fb20b0c8 Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sat, 8 Jan 2022 23:03:34 -0800 Subject: [PATCH 01/13] Adding initial support for bottlerocket without custom launch template --- .../provisioning/v1alpha5/requirements.go | 1 + pkg/cloudprovider/aws/ami.go | 16 +++++-- .../aws/apis/v1alpha1/provider.go | 3 ++ pkg/cloudprovider/aws/instancetype.go | 4 ++ pkg/cloudprovider/aws/launchtemplate.go | 42 ++++++++++++++++++- pkg/cloudprovider/fake/instancetype.go | 5 +++ pkg/cloudprovider/types.go | 1 + 7 files changed, 67 insertions(+), 5 deletions(-) diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go index 69bd27ea8b10..de259832d4a3 100644 --- a/pkg/apis/provisioning/v1alpha5/requirements.go +++ b/pkg/apis/provisioning/v1alpha5/requirements.go @@ -25,6 +25,7 @@ var ( ArchitectureAmd64 = "amd64" ArchitectureArm64 = "arm64" OperatingSystemLinux = "linux" + BottleRocket = "bottlerocket" // RestrictedLabels are injected by Cloud Providers RestrictedLabels = sets.NewString( diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 0a50bbc5d78f..95c79b83b922 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/patrickmn/go-cache" "k8s.io/client-go/kubernetes" "knative.dev/pkg/logging" @@ -46,7 +47,7 @@ func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIPr } // Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { +func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType, constraints *v1alpha1.Constraints) (map[string][]cloudprovider.InstanceType, error) { version, err := p.kubeServerVersion(ctx) if err != nil { return nil, fmt.Errorf("kube server version, %w", err) @@ -54,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(instanceType, version) + query := p.getSSMQuery(ctx, constraints.AMIFamily, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs @@ -83,12 +84,21 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(instanceType cloudprovider.InstanceType, version string) string { +func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanceType cloudprovider.InstanceType, version string) string { var amiSuffix string + var arch string if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { amiSuffix = "-gpu" } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { amiSuffix = "-arm64" + arch = "arm64" + } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { + arch = "arm64" + } + + if amiFamily == v1alpha5.BottleRocket { + logging.FromContext(ctx).Debugf("AMIFamily was: %s", amiFamily) + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) } return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 1ac259dc2b93..7cd37e97e169 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -34,6 +34,9 @@ type AWS struct { // TypeMeta includes version and kind of the extensions, inferred if not provided. // +optional metav1.TypeMeta `json:",inline"` + // AMIFamily is the AMI family that instances use. + // +optional + AMIFamily string `json:"amiFamily"` // InstanceProfile is the AWS identity that instances use. // +required InstanceProfile string `json:"instanceProfile"` diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 4c5b46ca1ba1..f5c2d92f8b9a 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -40,6 +40,10 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } +func (i *InstanceType) AMIFamily() string { + return "bottlerocket" +} + func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings } diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 9291127585d1..1f9f1772f6c1 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -94,7 +94,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. return nil, err } // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, instanceTypes) + amis, err := p.amiProvider.Get(ctx, instanceTypes, constraints) if err != nil { return nil, err } @@ -102,7 +102,12 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. launchTemplates := map[string][]cloudprovider.InstanceType{} for amiID, instanceTypes := range amis { // Get userData for Node - userData, err := p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + var userData string + if constraints.AMIFamily == "bottlerocket" { + userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } else { + userData, err = p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + } if err != nil { return nil, err } @@ -220,6 +225,39 @@ func sortedKeys(m map[string]string) []string { return keys } +func (p *LaunchTemplateProvider) getBottleRocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string) (string, error) { + var userData bytes.Buffer + userData.WriteString(fmt.Sprintf(`[settings.kubernetes] +cluster-name = "%s" +api-server = "%s" +`, + injection.GetOptions(ctx).ClusterName, + injection.GetOptions(ctx).ClusterEndpoint)) + caBundle, err := p.GetCABundle(ctx) + if err != nil { + return "", fmt.Errorf("getting ca bundle for user data, %w", err) + } + if caBundle != nil { + userData.WriteString(fmt.Sprintf(`cluster-certificate = "%s" + +`, + *caBundle)) + } + + nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) + if len(nodeLabelArgs) > 0 { + userData.WriteString(`[settings.kubernetes.node-labels] +`) + for key, val := range nodeLabelArgs { + userData.WriteString(fmt.Sprintf(`"%s" = "%s" +`, + key, val)) + } + } + + return base64.StdEncoding.EncodeToString(userData.Bytes()), nil +} + // getUserData returns the exact same string for equivalent input, // even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 84a1fa174041..98b456bb2ded 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -87,6 +87,7 @@ type InstanceTypeOptions struct { Name string Offerings []cloudprovider.Offering Architecture string + AMIFamily string OperatingSystems sets.String CPU resource.Quantity Memory resource.Quantity @@ -105,6 +106,10 @@ func (i *InstanceType) Name() string { return i.options.Name } +func (i *InstanceType) AMIFamily() string { + return "bottlerocket" +} + func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.options.Offerings } diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 88620e921a88..c7db0829f4a7 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -57,6 +57,7 @@ type InstanceType interface { // Note that though this is an array it is expected that all the Offerings are unique from one another Offerings() []Offering Architecture() string + AMIFamily() string OperatingSystems() sets.String CPU() *resource.Quantity Memory() *resource.Quantity From cd736bcdfbaf9a924b316b10eb44e319b93a8f03 Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sun, 9 Jan 2022 15:25:09 -0800 Subject: [PATCH 02/13] Fix small issue to allow both eks optimized and bottlerocket --- pkg/cloudprovider/aws/ami.go | 2 +- pkg/cloudprovider/aws/instancetype.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 95c79b83b922..371b5971f35d 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -55,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(ctx, constraints.AMIFamily, instanceType, version) + query := p.getSSMQuery(ctx, constraints.AWS.AMIFamily, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index f5c2d92f8b9a..0f0ffbf8eaf7 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -41,7 +41,7 @@ func (i *InstanceType) Name() string { } func (i *InstanceType) AMIFamily() string { - return "bottlerocket" + return "" } func (i *InstanceType) Offerings() []cloudprovider.Offering { From 6de2379ac40b5a0d7dd4afd0cae35ad92ab6a8be Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sat, 15 Jan 2022 11:06:48 -0800 Subject: [PATCH 03/13] Updates with requested changes --- .../provisioning/v1alpha5/requirements.go | 1 - pkg/cloudprovider/aws/ami.go | 15 ++++--- .../aws/apis/v1alpha1/provider.go | 2 +- .../aws/apis/v1alpha1/register.go | 1 + pkg/cloudprovider/aws/instancetype.go | 2 + pkg/cloudprovider/aws/launchtemplate.go | 41 ++++++++++--------- pkg/cloudprovider/types.go | 1 - 7 files changed, 35 insertions(+), 28 deletions(-) diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go index de259832d4a3..69bd27ea8b10 100644 --- a/pkg/apis/provisioning/v1alpha5/requirements.go +++ b/pkg/apis/provisioning/v1alpha5/requirements.go @@ -25,7 +25,6 @@ var ( ArchitectureAmd64 = "amd64" ArchitectureArm64 = "arm64" OperatingSystemLinux = "linux" - BottleRocket = "bottlerocket" // RestrictedLabels are injected by Cloud Providers RestrictedLabels = sets.NewString( diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 371b5971f35d..39598b2320b9 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -47,7 +47,7 @@ func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIPr } // Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType, constraints *v1alpha1.Constraints) (map[string][]cloudprovider.InstanceType, error) { +func (p *AMIProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { version, err := p.kubeServerVersion(ctx) if err != nil { return nil, fmt.Errorf("kube server version, %w", err) @@ -55,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(ctx, constraints.AWS.AMIFamily, instanceType, version) + query := p.getSSMQuery(ctx, constraints, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs @@ -84,7 +84,7 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanceType cloudprovider.InstanceType, version string) string { +func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { var amiSuffix string var arch string if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { @@ -96,10 +96,13 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanc arch = "arm64" } - if amiFamily == v1alpha5.BottleRocket { - logging.FromContext(ctx).Debugf("AMIFamily was: %s", amiFamily) - return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + if constraints.AMIFamily != nil { + switch *constraints.AMIFamily { + case v1alpha1.OperatingSystemBottleRocket: + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + } } + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 7cd37e97e169..82a0e310e5a5 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -36,7 +36,7 @@ type AWS struct { metav1.TypeMeta `json:",inline"` // AMIFamily is the AMI family that instances use. // +optional - AMIFamily string `json:"amiFamily"` + AMIFamily *string `json:"amiFamily,omitempty"` // InstanceProfile is the AWS identity that instances use. // +required InstanceProfile string `json:"instanceProfile"` diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 4adba7c0946e..6a2158db86a4 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,6 +32,7 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } + OperatingSystemBottleRocket = "bottlerocket" ) var ( diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 0f0ffbf8eaf7..976f5ff5b611 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -40,9 +40,11 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } +/* func (i *InstanceType) AMIFamily() string { return "" } +*/ func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 1f9f1772f6c1..238e9b8745e4 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -94,7 +94,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. return nil, err } // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, instanceTypes, constraints) + amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes) if err != nil { return nil, err } @@ -103,10 +103,13 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. for amiID, instanceTypes := range amis { // Get userData for Node var userData string - if constraints.AMIFamily == "bottlerocket" { - userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + if constraints.AMIFamily != nil { + switch *constraints.AMIFamily { + case v1alpha1.OperatingSystemBottleRocket: + userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } } else { - userData, err = p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } if err != nil { return nil, err @@ -226,42 +229,42 @@ func sortedKeys(m map[string]string) []string { } func (p *LaunchTemplateProvider) getBottleRocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string) (string, error) { - var userData bytes.Buffer - userData.WriteString(fmt.Sprintf(`[settings.kubernetes] + var userData string + userData += fmt.Sprintf(`[settings.kubernetes] cluster-name = "%s" api-server = "%s" `, injection.GetOptions(ctx).ClusterName, - injection.GetOptions(ctx).ClusterEndpoint)) + injection.GetOptions(ctx).ClusterEndpoint) + + if constraints.KubeletConfiguration.ClusterDNS != nil { + userData += fmt.Sprintf("cluster-dns-ip = \"%s\"", constraints.KubeletConfiguration.ClusterDNS) + } + caBundle, err := p.GetCABundle(ctx) if err != nil { return "", fmt.Errorf("getting ca bundle for user data, %w", err) } if caBundle != nil { - userData.WriteString(fmt.Sprintf(`cluster-certificate = "%s" - -`, - *caBundle)) + userData += fmt.Sprintf("cluster-certificate = \"%s\"\n", *caBundle) } nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) if len(nodeLabelArgs) > 0 { - userData.WriteString(`[settings.kubernetes.node-labels] -`) + userData += `[settings.kubernetes.node-labels] +` for key, val := range nodeLabelArgs { - userData.WriteString(fmt.Sprintf(`"%s" = "%s" -`, - key, val)) + userData += fmt.Sprintf("\"%s\" = \"%s\"", key, val) } } - return base64.StdEncoding.EncodeToString(userData.Bytes()), nil + return base64.StdEncoding.EncodeToString([]byte(userData)), nil } -// getUserData returns the exact same string for equivalent input, +// getEKSOptimizedUserData returns the exact same string for equivalent input, // even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. -func (p *LaunchTemplateProvider) getUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { +func (p *LaunchTemplateProvider) getEKSOptimizedUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { var containerRuntimeArg string if !needsDocker(instanceTypes) { containerRuntimeArg = "--container-runtime containerd" diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index c7db0829f4a7..88620e921a88 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -57,7 +57,6 @@ type InstanceType interface { // Note that though this is an array it is expected that all the Offerings are unique from one another Offerings() []Offering Architecture() string - AMIFamily() string OperatingSystems() sets.String CPU() *resource.Quantity Memory() *resource.Quantity From 0f8cafa16a6ef6868ebb0cfb332e92606efbd82a Mon Sep 17 00:00:00 2001 From: rayterrill Date: Thu, 20 Jan 2022 08:50:30 -0800 Subject: [PATCH 04/13] Update --- pkg/cloudprovider/aws/ami.go | 11 +++++++---- pkg/cloudprovider/aws/apis/v1alpha1/register.go | 3 ++- pkg/cloudprovider/aws/instancetype.go | 6 ------ pkg/cloudprovider/aws/launchtemplate.go | 7 +++++-- pkg/cloudprovider/fake/instancetype.go | 5 ----- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 39598b2320b9..5be44ff9e38e 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -97,13 +97,16 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Con } if constraints.AMIFamily != nil { - switch *constraints.AMIFamily { - case v1alpha1.OperatingSystemBottleRocket: + if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + } else { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } + } else { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } - - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 6a2158db86a4..b0ce93fb50d3 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,7 +32,8 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } - OperatingSystemBottleRocket = "bottlerocket" + OperatingSystemBottleRocket = "BottleRocket" + OperatingSystemEKSOptimized = "EKSOptimized" ) var ( diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 976f5ff5b611..4c5b46ca1ba1 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -40,12 +40,6 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } -/* -func (i *InstanceType) AMIFamily() string { - return "" -} -*/ - func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings } diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 238e9b8745e4..bebbd4aa9c23 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -104,9 +104,12 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. // Get userData for Node var userData string if constraints.AMIFamily != nil { - switch *constraints.AMIFamily { - case v1alpha1.OperatingSystemBottleRocket: + if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) + } else { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } } else { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 98b456bb2ded..84a1fa174041 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -87,7 +87,6 @@ type InstanceTypeOptions struct { Name string Offerings []cloudprovider.Offering Architecture string - AMIFamily string OperatingSystems sets.String CPU resource.Quantity Memory resource.Quantity @@ -106,10 +105,6 @@ func (i *InstanceType) Name() string { return i.options.Name } -func (i *InstanceType) AMIFamily() string { - return "bottlerocket" -} - func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.options.Offerings } From e6bc0e795412573e3fc1057a07f4680c43a84cca Mon Sep 17 00:00:00 2001 From: Ray Terrill Date: Tue, 25 Jan 2022 19:53:01 -0800 Subject: [PATCH 05/13] Updates --- pkg/cloudprovider/aws/ami.go | 2 +- pkg/cloudprovider/aws/apis/v1alpha1/register.go | 2 +- pkg/cloudprovider/aws/launchtemplate.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 5be44ff9e38e..f50a36fd85b0 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -93,7 +93,7 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Con amiSuffix = "-arm64" arch = "arm64" } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { - arch = "arm64" + arch = "x86_64" } if constraints.AMIFamily != nil { diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index b0ce93fb50d3..3562efa7ac9c 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,7 +32,7 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } - OperatingSystemBottleRocket = "BottleRocket" + OperatingSystemBottleRocket = "Bottlerocket" OperatingSystemEKSOptimized = "EKSOptimized" ) diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index bebbd4aa9c23..a18b08ae7a03 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -104,9 +104,9 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. // Get userData for Node var userData string if constraints.AMIFamily != nil { - if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { + if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemBottleRocket) { userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) - } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + } else if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemEKSOptimized) { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } else { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) From ee7421068a3698677c252376f7f3cc1cb0b4ac77 Mon Sep 17 00:00:00 2001 From: Ray Terrill Date: Sat, 29 Jan 2022 14:24:24 -0800 Subject: [PATCH 06/13] Update --- pkg/cloudprovider/aws/ami.go | 35 ++++++++++++++++--------- pkg/cloudprovider/aws/launchtemplate.go | 10 +++++++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index f50a36fd85b0..4e3f735e9a58 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -84,29 +84,40 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { - var amiSuffix string - var arch string +// getAL2Alias returns a properly-formatted alias for an Amazon Linux AMI from SSM +func (p *AMIProvider) getAL2Alias(version string, instanceType cloudprovider.InstanceType) string { + amiSuffix := "" if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { amiSuffix = "-gpu" } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { - amiSuffix = "-arm64" - arch = "arm64" - } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { - arch = "x86_64" + amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture()) } + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) +} +// getBottlerocketAlias returns a properly-formatted alias for a Bottlerocket AMI from SSM +func (p *AMIProvider) getBottlerocketAlias(version string, instanceType cloudprovider.InstanceType) string { + arch := "x86_64" + if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { + arch = instanceType.Architecture() + } + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) +} + +func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { + ssmQuery := p.getAL2Alias(version, instanceType) if constraints.AMIFamily != nil { if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { - return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + ssmQuery = p.getBottlerocketAlias(version, instanceType) } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + ssmQuery = p.getAL2Alias(version, instanceType) } else { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) + ssmQuery = p.getAL2Alias(version, instanceType) } - } else { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } + + return ssmQuery } func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index a18b08ae7a03..0e5a0a6e67d6 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -109,6 +109,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. } else if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemEKSOptimized) { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } else { + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } } else { @@ -261,6 +262,15 @@ api-server = "%s" } } + if len(constraints.Taints) > 0 { + userData += `[settings.kubernetes.node-taints] +` + sorted := sortedTaints(constraints.Taints) + for _, taint := range sorted { + userData += fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect) + } + } + return base64.StdEncoding.EncodeToString([]byte(userData)), nil } From 7640e6b8f3fe5acf6d6e740debf618a067dc06ff Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sat, 8 Jan 2022 23:03:34 -0800 Subject: [PATCH 07/13] Adding initial support for bottlerocket without custom launch template --- .../provisioning/v1alpha5/requirements.go | 1 + pkg/cloudprovider/aws/ami.go | 16 +++++-- .../aws/apis/v1alpha1/provider.go | 5 ++- pkg/cloudprovider/aws/instancetype.go | 4 ++ pkg/cloudprovider/aws/launchtemplate.go | 42 ++++++++++++++++++- pkg/cloudprovider/fake/instancetype.go | 5 +++ pkg/cloudprovider/types.go | 1 + 7 files changed, 68 insertions(+), 6 deletions(-) diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go index 49d950d185cf..ef581040d2f3 100644 --- a/pkg/apis/provisioning/v1alpha5/requirements.go +++ b/pkg/apis/provisioning/v1alpha5/requirements.go @@ -30,6 +30,7 @@ var ( ArchitectureAmd64 = "amd64" ArchitectureArm64 = "arm64" OperatingSystemLinux = "linux" + BottleRocket = "bottlerocket" // RestrictedLabels are injected by Cloud Providers RestrictedLabels = stringsets.NewString( diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 0a50bbc5d78f..95c79b83b922 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/patrickmn/go-cache" "k8s.io/client-go/kubernetes" "knative.dev/pkg/logging" @@ -46,7 +47,7 @@ func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIPr } // Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { +func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType, constraints *v1alpha1.Constraints) (map[string][]cloudprovider.InstanceType, error) { version, err := p.kubeServerVersion(ctx) if err != nil { return nil, fmt.Errorf("kube server version, %w", err) @@ -54,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(instanceType, version) + query := p.getSSMQuery(ctx, constraints.AMIFamily, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs @@ -83,12 +84,21 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(instanceType cloudprovider.InstanceType, version string) string { +func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanceType cloudprovider.InstanceType, version string) string { var amiSuffix string + var arch string if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { amiSuffix = "-gpu" } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { amiSuffix = "-arm64" + arch = "arm64" + } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { + arch = "arm64" + } + + if amiFamily == v1alpha5.BottleRocket { + logging.FromContext(ctx).Debugf("AMIFamily was: %s", amiFamily) + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) } return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 3dd3c6c2e216..1282f844635d 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -43,8 +43,11 @@ type AWS struct { // TypeMeta includes version and kind of the extensions, inferred if not provided. // +optional metav1.TypeMeta `json:",inline"` - // InstanceProfile to use for provisioned nodes, overriding the default profile. + // AMIFamily is the AMI family that instances use. // +optional + AMIFamily string `json:"amiFamily"` + // InstanceProfile is the AWS identity that instances use. + // +required InstanceProfile string `json:"instanceProfile"` // LaunchTemplate for the node. If not specified, a launch template will be generated. // +optional diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 43c615088a68..3da4daa56525 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -41,6 +41,10 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } +func (i *InstanceType) AMIFamily() string { + return "bottlerocket" +} + func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings } diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index fcc99706e717..adacf664c1d4 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -100,7 +100,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. return nil, err } // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, instanceTypes) + amis, err := p.amiProvider.Get(ctx, instanceTypes, constraints) if err != nil { return nil, err } @@ -108,7 +108,12 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. launchTemplates := map[string][]cloudprovider.InstanceType{} for amiID, instanceTypes := range amis { // Get userData for Node - userData, err := p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + var userData string + if constraints.AMIFamily == "bottlerocket" { + userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } else { + userData, err = p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + } if err != nil { return nil, err } @@ -240,6 +245,39 @@ func sortedKeys(m map[string]string) []string { return keys } +func (p *LaunchTemplateProvider) getBottleRocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string) (string, error) { + var userData bytes.Buffer + userData.WriteString(fmt.Sprintf(`[settings.kubernetes] +cluster-name = "%s" +api-server = "%s" +`, + injection.GetOptions(ctx).ClusterName, + injection.GetOptions(ctx).ClusterEndpoint)) + caBundle, err := p.GetCABundle(ctx) + if err != nil { + return "", fmt.Errorf("getting ca bundle for user data, %w", err) + } + if caBundle != nil { + userData.WriteString(fmt.Sprintf(`cluster-certificate = "%s" + +`, + *caBundle)) + } + + nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) + if len(nodeLabelArgs) > 0 { + userData.WriteString(`[settings.kubernetes.node-labels] +`) + for key, val := range nodeLabelArgs { + userData.WriteString(fmt.Sprintf(`"%s" = "%s" +`, + key, val)) + } + } + + return base64.StdEncoding.EncodeToString(userData.Bytes()), nil +} + // getUserData returns the exact same string for equivalent input, // even if elements of those inputs are in differeing orders, // guaranteeing it won't cause spurious hash differences. diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index c7a47be70e54..197a8308dec4 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -86,6 +86,7 @@ type InstanceTypeOptions struct { Name string Offerings []cloudprovider.Offering Architecture string + AMIFamily string OperatingSystems sets.String CPU resource.Quantity Memory resource.Quantity @@ -104,6 +105,10 @@ func (i *InstanceType) Name() string { return i.options.Name } +func (i *InstanceType) AMIFamily() string { + return "bottlerocket" +} + func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.options.Offerings } diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index af23ea61714c..7767e971441d 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -57,6 +57,7 @@ type InstanceType interface { // Note that though this is an array it is expected that all the Offerings are unique from one another Offerings() []Offering Architecture() string + AMIFamily() string OperatingSystems() sets.String CPU() *resource.Quantity Memory() *resource.Quantity From dbeddfa837c4e58a7bfa4418bfd1230e1f19824e Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sun, 9 Jan 2022 15:25:09 -0800 Subject: [PATCH 08/13] Fix small issue to allow both eks optimized and bottlerocket --- pkg/cloudprovider/aws/ami.go | 2 +- pkg/cloudprovider/aws/instancetype.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 95c79b83b922..371b5971f35d 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -55,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(ctx, constraints.AMIFamily, instanceType, version) + query := p.getSSMQuery(ctx, constraints.AWS.AMIFamily, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 3da4daa56525..7455e2d64cfc 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -42,7 +42,7 @@ func (i *InstanceType) Name() string { } func (i *InstanceType) AMIFamily() string { - return "bottlerocket" + return "" } func (i *InstanceType) Offerings() []cloudprovider.Offering { From 6818aff4f151fb677e402db66901764b658eb19b Mon Sep 17 00:00:00 2001 From: rayterrill Date: Sat, 15 Jan 2022 11:06:48 -0800 Subject: [PATCH 09/13] Updates with requested changes --- .../provisioning/v1alpha5/requirements.go | 1 - pkg/cloudprovider/aws/ami.go | 15 ++++--- .../aws/apis/v1alpha1/provider.go | 2 +- .../aws/apis/v1alpha1/register.go | 1 + pkg/cloudprovider/aws/instancetype.go | 2 + pkg/cloudprovider/aws/launchtemplate.go | 43 ++++++++++--------- pkg/cloudprovider/types.go | 1 - 7 files changed, 36 insertions(+), 29 deletions(-) diff --git a/pkg/apis/provisioning/v1alpha5/requirements.go b/pkg/apis/provisioning/v1alpha5/requirements.go index ef581040d2f3..49d950d185cf 100644 --- a/pkg/apis/provisioning/v1alpha5/requirements.go +++ b/pkg/apis/provisioning/v1alpha5/requirements.go @@ -30,7 +30,6 @@ var ( ArchitectureAmd64 = "amd64" ArchitectureArm64 = "arm64" OperatingSystemLinux = "linux" - BottleRocket = "bottlerocket" // RestrictedLabels are injected by Cloud Providers RestrictedLabels = stringsets.NewString( diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 371b5971f35d..39598b2320b9 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -47,7 +47,7 @@ func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIPr } // Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType, constraints *v1alpha1.Constraints) (map[string][]cloudprovider.InstanceType, error) { +func (p *AMIProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { version, err := p.kubeServerVersion(ctx) if err != nil { return nil, fmt.Errorf("kube server version, %w", err) @@ -55,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(ctx, constraints.AWS.AMIFamily, instanceType, version) + query := p.getSSMQuery(ctx, constraints, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs @@ -84,7 +84,7 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanceType cloudprovider.InstanceType, version string) string { +func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { var amiSuffix string var arch string if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { @@ -96,10 +96,13 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, amiFamily string, instanc arch = "arm64" } - if amiFamily == v1alpha5.BottleRocket { - logging.FromContext(ctx).Debugf("AMIFamily was: %s", amiFamily) - return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + if constraints.AMIFamily != nil { + switch *constraints.AMIFamily { + case v1alpha1.OperatingSystemBottleRocket: + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + } } + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 1282f844635d..1e8f00c49a73 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -45,7 +45,7 @@ type AWS struct { metav1.TypeMeta `json:",inline"` // AMIFamily is the AMI family that instances use. // +optional - AMIFamily string `json:"amiFamily"` + AMIFamily *string `json:"amiFamily,omitempty"` // InstanceProfile is the AWS identity that instances use. // +required InstanceProfile string `json:"instanceProfile"` diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 4adba7c0946e..6a2158db86a4 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,6 +32,7 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } + OperatingSystemBottleRocket = "bottlerocket" ) var ( diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 7455e2d64cfc..29c567fdf2c8 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -41,9 +41,11 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } +/* func (i *InstanceType) AMIFamily() string { return "" } +*/ func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index adacf664c1d4..9aea27c443b3 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -100,7 +100,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. return nil, err } // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, instanceTypes, constraints) + amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes) if err != nil { return nil, err } @@ -109,10 +109,13 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. for amiID, instanceTypes := range amis { // Get userData for Node var userData string - if constraints.AMIFamily == "bottlerocket" { - userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + if constraints.AMIFamily != nil { + switch *constraints.AMIFamily { + case v1alpha1.OperatingSystemBottleRocket: + userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } } else { - userData, err = p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } if err != nil { return nil, err @@ -246,42 +249,42 @@ func sortedKeys(m map[string]string) []string { } func (p *LaunchTemplateProvider) getBottleRocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string) (string, error) { - var userData bytes.Buffer - userData.WriteString(fmt.Sprintf(`[settings.kubernetes] + var userData string + userData += fmt.Sprintf(`[settings.kubernetes] cluster-name = "%s" api-server = "%s" `, injection.GetOptions(ctx).ClusterName, - injection.GetOptions(ctx).ClusterEndpoint)) + injection.GetOptions(ctx).ClusterEndpoint) + + if constraints.KubeletConfiguration.ClusterDNS != nil { + userData += fmt.Sprintf("cluster-dns-ip = \"%s\"", constraints.KubeletConfiguration.ClusterDNS) + } + caBundle, err := p.GetCABundle(ctx) if err != nil { return "", fmt.Errorf("getting ca bundle for user data, %w", err) } if caBundle != nil { - userData.WriteString(fmt.Sprintf(`cluster-certificate = "%s" - -`, - *caBundle)) + userData += fmt.Sprintf("cluster-certificate = \"%s\"\n", *caBundle) } nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) if len(nodeLabelArgs) > 0 { - userData.WriteString(`[settings.kubernetes.node-labels] -`) + userData += `[settings.kubernetes.node-labels] +` for key, val := range nodeLabelArgs { - userData.WriteString(fmt.Sprintf(`"%s" = "%s" -`, - key, val)) + userData += fmt.Sprintf("\"%s\" = \"%s\"", key, val) } } - return base64.StdEncoding.EncodeToString(userData.Bytes()), nil + return base64.StdEncoding.EncodeToString([]byte(userData)), nil } -// getUserData returns the exact same string for equivalent input, -// even if elements of those inputs are in differeing orders, +// getEKSOptimizedUserData returns the exact same string for equivalent input, +// even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. -func (p *LaunchTemplateProvider) getUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { +func (p *LaunchTemplateProvider) getEKSOptimizedUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { var containerRuntimeArg string if !needsDocker(instanceTypes) { containerRuntimeArg = "--container-runtime containerd" diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 7767e971441d..af23ea61714c 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -57,7 +57,6 @@ type InstanceType interface { // Note that though this is an array it is expected that all the Offerings are unique from one another Offerings() []Offering Architecture() string - AMIFamily() string OperatingSystems() sets.String CPU() *resource.Quantity Memory() *resource.Quantity From 64b650adf1d91208593a95dc5d86f10691666a77 Mon Sep 17 00:00:00 2001 From: rayterrill Date: Thu, 20 Jan 2022 08:50:30 -0800 Subject: [PATCH 10/13] Update --- pkg/cloudprovider/aws/ami.go | 11 +++++++---- pkg/cloudprovider/aws/apis/v1alpha1/register.go | 3 ++- pkg/cloudprovider/aws/instancetype.go | 6 ------ pkg/cloudprovider/aws/launchtemplate.go | 7 +++++-- pkg/cloudprovider/fake/instancetype.go | 5 ----- 5 files changed, 14 insertions(+), 18 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 39598b2320b9..5be44ff9e38e 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -97,13 +97,16 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Con } if constraints.AMIFamily != nil { - switch *constraints.AMIFamily { - case v1alpha1.OperatingSystemBottleRocket: + if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + } else { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } + } else { + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } - - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 6a2158db86a4..b0ce93fb50d3 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,7 +32,8 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } - OperatingSystemBottleRocket = "bottlerocket" + OperatingSystemBottleRocket = "BottleRocket" + OperatingSystemEKSOptimized = "EKSOptimized" ) var ( diff --git a/pkg/cloudprovider/aws/instancetype.go b/pkg/cloudprovider/aws/instancetype.go index 29c567fdf2c8..43c615088a68 100644 --- a/pkg/cloudprovider/aws/instancetype.go +++ b/pkg/cloudprovider/aws/instancetype.go @@ -41,12 +41,6 @@ func (i *InstanceType) Name() string { return aws.StringValue(i.InstanceType) } -/* -func (i *InstanceType) AMIFamily() string { - return "" -} -*/ - func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.AvailableOfferings } diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 9aea27c443b3..ca199f29c19e 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -110,9 +110,12 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. // Get userData for Node var userData string if constraints.AMIFamily != nil { - switch *constraints.AMIFamily { - case v1alpha1.OperatingSystemBottleRocket: + if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) + } else { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } } else { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) diff --git a/pkg/cloudprovider/fake/instancetype.go b/pkg/cloudprovider/fake/instancetype.go index 197a8308dec4..c7a47be70e54 100644 --- a/pkg/cloudprovider/fake/instancetype.go +++ b/pkg/cloudprovider/fake/instancetype.go @@ -86,7 +86,6 @@ type InstanceTypeOptions struct { Name string Offerings []cloudprovider.Offering Architecture string - AMIFamily string OperatingSystems sets.String CPU resource.Quantity Memory resource.Quantity @@ -105,10 +104,6 @@ func (i *InstanceType) Name() string { return i.options.Name } -func (i *InstanceType) AMIFamily() string { - return "bottlerocket" -} - func (i *InstanceType) Offerings() []cloudprovider.Offering { return i.options.Offerings } From e30936caa953ae78dd5a37c0c11f31fcf68ca929 Mon Sep 17 00:00:00 2001 From: Ray Terrill Date: Tue, 25 Jan 2022 19:53:01 -0800 Subject: [PATCH 11/13] Updates --- pkg/cloudprovider/aws/ami.go | 2 +- pkg/cloudprovider/aws/apis/v1alpha1/register.go | 2 +- pkg/cloudprovider/aws/launchtemplate.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 5be44ff9e38e..f50a36fd85b0 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -93,7 +93,7 @@ func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Con amiSuffix = "-arm64" arch = "arm64" } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { - arch = "arm64" + arch = "x86_64" } if constraints.AMIFamily != nil { diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index b0ce93fb50d3..3562efa7ac9c 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,7 +32,7 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } - OperatingSystemBottleRocket = "BottleRocket" + OperatingSystemBottleRocket = "Bottlerocket" OperatingSystemEKSOptimized = "EKSOptimized" ) diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index ca199f29c19e..8c87c6f39076 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -110,9 +110,9 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. // Get userData for Node var userData string if constraints.AMIFamily != nil { - if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { + if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemBottleRocket) { userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) - } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + } else if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemEKSOptimized) { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } else { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) From b504aeec7ee6eb0795f9ea1fda853cbaf4a4bb2f Mon Sep 17 00:00:00 2001 From: Ray Terrill Date: Sat, 29 Jan 2022 14:24:24 -0800 Subject: [PATCH 12/13] Update --- pkg/cloudprovider/aws/ami.go | 35 ++++++++++++++++--------- pkg/cloudprovider/aws/launchtemplate.go | 10 +++++++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index f50a36fd85b0..4e3f735e9a58 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -84,29 +84,40 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { - var amiSuffix string - var arch string +// getAL2Alias returns a properly-formatted alias for an Amazon Linux AMI from SSM +func (p *AMIProvider) getAL2Alias(version string, instanceType cloudprovider.InstanceType) string { + amiSuffix := "" if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { amiSuffix = "-gpu" } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { - amiSuffix = "-arm64" - arch = "arm64" - } else if instanceType.Architecture() == v1alpha5.ArchitectureAmd64 { - arch = "x86_64" + amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture()) } + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) +} +// getBottlerocketAlias returns a properly-formatted alias for a Bottlerocket AMI from SSM +func (p *AMIProvider) getBottlerocketAlias(version string, instanceType cloudprovider.InstanceType) string { + arch := "x86_64" + if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { + arch = instanceType.Architecture() + } + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) +} + +func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { + ssmQuery := p.getAL2Alias(version, instanceType) if constraints.AMIFamily != nil { if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { - return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) + ssmQuery = p.getBottlerocketAlias(version, instanceType) } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + ssmQuery = p.getAL2Alias(version, instanceType) } else { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) + ssmQuery = p.getAL2Alias(version, instanceType) } - } else { - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } + + return ssmQuery } func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 8c87c6f39076..481c946c7946 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -115,6 +115,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. } else if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemEKSOptimized) { userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } else { + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) } } else { @@ -281,6 +282,15 @@ api-server = "%s" } } + if len(constraints.Taints) > 0 { + userData += `[settings.kubernetes.node-taints] +` + sorted := sortedTaints(constraints.Taints) + for _, taint := range sorted { + userData += fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect) + } + } + return base64.StdEncoding.EncodeToString([]byte(userData)), nil } From a89b7beb6effafaabb570bb3a9ccecded3389b27 Mon Sep 17 00:00:00 2001 From: Ray Terrill Date: Tue, 8 Feb 2022 08:12:53 -0800 Subject: [PATCH 13/13] Update with codegen --- pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go index ec030183d921..eadd0c255924 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,11 @@ import ( func (in *AWS) DeepCopyInto(out *AWS) { *out = *in out.TypeMeta = in.TypeMeta + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } if in.LaunchTemplate != nil { in, out := &in.LaunchTemplate, &out.LaunchTemplate *out = new(string)