diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index 0a50bbc5d78f..4e3f735e9a58 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/service/ssm/ssmiface" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/patrickmn/go-cache" "k8s.io/client-go/kubernetes" "knative.dev/pkg/logging" @@ -46,7 +47,7 @@ func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIPr } // Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { +func (p *AMIProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { version, err := p.kubeServerVersion(ctx) if err != nil { return nil, fmt.Errorf("kube server version, %w", err) @@ -54,7 +55,7 @@ func (p *AMIProvider) Get(ctx context.Context, instanceTypes []cloudprovider.Ins // Separate instance types by unique queries amiQueries := map[string][]cloudprovider.InstanceType{} for _, instanceType := range instanceTypes { - query := p.getSSMQuery(instanceType, version) + query := p.getSSMQuery(ctx, constraints, instanceType, version) amiQueries[query] = append(amiQueries[query], instanceType) } // Separate instance types by unique AMIIDs @@ -83,16 +84,42 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error return ami, nil } -func (p *AMIProvider) getSSMQuery(instanceType cloudprovider.InstanceType, version string) string { - var amiSuffix string +// getAL2Alias returns a properly-formatted alias for an Amazon Linux AMI from SSM +func (p *AMIProvider) getAL2Alias(version string, instanceType cloudprovider.InstanceType) string { + amiSuffix := "" if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { amiSuffix = "-gpu" } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { - amiSuffix = "-arm64" + amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture()) } return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) } +// getBottlerocketAlias returns a properly-formatted alias for a Bottlerocket AMI from SSM +func (p *AMIProvider) getBottlerocketAlias(version string, instanceType cloudprovider.InstanceType) string { + arch := "x86_64" + if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { + arch = instanceType.Architecture() + } + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) +} + +func (p *AMIProvider) getSSMQuery(ctx context.Context, constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { + ssmQuery := p.getAL2Alias(version, instanceType) + if constraints.AMIFamily != nil { + if *constraints.AMIFamily == v1alpha1.OperatingSystemBottleRocket { + ssmQuery = p.getBottlerocketAlias(version, instanceType) + } else if *constraints.AMIFamily == v1alpha1.OperatingSystemEKSOptimized { + ssmQuery = p.getAL2Alias(version, instanceType) + } else { + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) + ssmQuery = p.getAL2Alias(version, instanceType) + } + } + + return ssmQuery +} + func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { return version.(string), nil diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 3dd3c6c2e216..1e8f00c49a73 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -43,8 +43,11 @@ type AWS struct { // TypeMeta includes version and kind of the extensions, inferred if not provided. // +optional metav1.TypeMeta `json:",inline"` - // InstanceProfile to use for provisioned nodes, overriding the default profile. + // AMIFamily is the AMI family that instances use. // +optional + AMIFamily *string `json:"amiFamily,omitempty"` + // InstanceProfile is the AWS identity that instances use. + // +required InstanceProfile string `json:"instanceProfile"` // LaunchTemplate for the node. If not specified, a launch template will be generated. // +optional diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 4adba7c0946e..3562efa7ac9c 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -32,6 +32,8 @@ var ( AWSRestrictedLabelDomains = []string{ "k8s.aws", } + OperatingSystemBottleRocket = "Bottlerocket" + OperatingSystemEKSOptimized = "EKSOptimized" ) var ( diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go index ec030183d921..eadd0c255924 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,11 @@ import ( func (in *AWS) DeepCopyInto(out *AWS) { *out = *in out.TypeMeta = in.TypeMeta + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } if in.LaunchTemplate != nil { in, out := &in.LaunchTemplate, &out.LaunchTemplate *out = new(string) diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index fcc99706e717..481c946c7946 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -100,7 +100,7 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. return nil, err } // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, instanceTypes) + amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes) if err != nil { return nil, err } @@ -108,7 +108,19 @@ func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1. launchTemplates := map[string][]cloudprovider.InstanceType{} for amiID, instanceTypes := range amis { // Get userData for Node - userData, err := p.getUserData(ctx, constraints, instanceTypes, additionalLabels) + var userData string + if constraints.AMIFamily != nil { + if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemBottleRocket) { + userData, err = p.getBottleRocketUserData(ctx, constraints, additionalLabels) + } else if strings.EqualFold(*constraints.AMIFamily, v1alpha1.OperatingSystemEKSOptimized) { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) + } else { + logging.FromContext(ctx).Warnf("AMIFamily was set, but was not one of %s or %s. Setting to %s as the default.", v1alpha1.OperatingSystemEKSOptimized, v1alpha1.OperatingSystemBottleRocket, v1alpha1.OperatingSystemEKSOptimized) + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) + } + } else { + userData, err = p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels) + } if err != nil { return nil, err } @@ -240,10 +252,52 @@ func sortedKeys(m map[string]string) []string { return keys } -// getUserData returns the exact same string for equivalent input, -// even if elements of those inputs are in differeing orders, +func (p *LaunchTemplateProvider) getBottleRocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string) (string, error) { + var userData string + userData += fmt.Sprintf(`[settings.kubernetes] +cluster-name = "%s" +api-server = "%s" +`, + injection.GetOptions(ctx).ClusterName, + injection.GetOptions(ctx).ClusterEndpoint) + + if constraints.KubeletConfiguration.ClusterDNS != nil { + userData += fmt.Sprintf("cluster-dns-ip = \"%s\"", constraints.KubeletConfiguration.ClusterDNS) + } + + caBundle, err := p.GetCABundle(ctx) + if err != nil { + return "", fmt.Errorf("getting ca bundle for user data, %w", err) + } + if caBundle != nil { + userData += fmt.Sprintf("cluster-certificate = \"%s\"\n", *caBundle) + } + + nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) + if len(nodeLabelArgs) > 0 { + userData += `[settings.kubernetes.node-labels] +` + for key, val := range nodeLabelArgs { + userData += fmt.Sprintf("\"%s\" = \"%s\"", key, val) + } + } + + if len(constraints.Taints) > 0 { + userData += `[settings.kubernetes.node-taints] +` + sorted := sortedTaints(constraints.Taints) + for _, taint := range sorted { + userData += fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect) + } + } + + return base64.StdEncoding.EncodeToString([]byte(userData)), nil +} + +// getEKSOptimizedUserData returns the exact same string for equivalent input, +// even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. -func (p *LaunchTemplateProvider) getUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { +func (p *LaunchTemplateProvider) getEKSOptimizedUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (string, error) { var containerRuntimeArg string if !needsDocker(instanceTypes) { containerRuntimeArg = "--container-runtime containerd"