diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go index c0126c69883e..0b507fb424dc 100644 --- a/pkg/cloudprovider/aws/ami.go +++ b/pkg/cloudprovider/aws/ami.go @@ -80,13 +80,16 @@ func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error } ami := aws.StringValue(output.Parameter.Value) p.cache.SetDefault(query, ami) - logging.FromContext(ctx).Debugf("Discovered ami %s for query %s", ami, query) + logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, query) return ami, nil } func (p *AMIProvider) getSSMQuery(constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { - if aws.StringValue(constraints.AMIFamily) == v1alpha1.AMIFamilyBottlerocket { + switch aws.StringValue(constraints.AMIFamily) { + case v1alpha1.AMIFamilyBottlerocket: return p.getBottlerocketAlias(version, instanceType) + case v1alpha1.AMIFamilyUbuntu: + return p.getUbuntuAlias(version, instanceType) } return p.getAL2Alias(version, instanceType) } @@ -111,6 +114,11 @@ func (p *AMIProvider) getBottlerocketAlias(version string, instanceType cloudpro return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/%s/latest/image_id", version, arch) } +// getUbuntuAlias returns a properly-formatted alias for an Ubuntu AMI from SSM +func (p *AMIProvider) getUbuntuAlias(version string, instanceType cloudprovider.InstanceType) string { + return fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, instanceType.Architecture()) +} + func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { return version.(string), nil diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 1e8f00c49a73..53c0add151af 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -47,8 +47,8 @@ type AWS struct { // +optional AMIFamily *string `json:"amiFamily,omitempty"` // InstanceProfile is the AWS identity that instances use. - // +required - InstanceProfile string `json:"instanceProfile"` + // +optional + InstanceProfile *string `json:"instanceProfile,omitempty"` // LaunchTemplate for the node. If not specified, a launch template will be generated. // +optional LaunchTemplate *string `json:"launchTemplate,omitempty"` diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go index 276072563b0a..13b8d17ec115 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go @@ -60,5 +60,5 @@ func (c *Constraints) defaultAMIFamily() { if c.AMIFamily != nil { return } - c.AMIFamily = &AMIFamilyEKSOptimized + c.AMIFamily = &AMIFamilyAL2 } diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go index 2f26c3339905..317856a6bd14 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go @@ -22,6 +22,15 @@ import ( "knative.dev/pkg/apis" ) +const ( + launchTemplatePath = "launchTemplate" + securityGroupSelectorPath = "securityGroupSelector" + fieldPathSubnetSelectorPath = "subnetSelector" + amiFamilyPath = "amiFamily" + metadataOptionsPath = "metadataOptions" + instanceProfilePath = "instanceProfile" +) + func (a *AWS) Validate() (errs *apis.FieldError) { return a.validate().ViaField("provider") } @@ -38,29 +47,46 @@ func (a *AWS) validate() (errs *apis.FieldError) { } func (a *AWS) validateLaunchTemplate() (errs *apis.FieldError) { - // nothing to validate at the moment + if a.LaunchTemplate == nil { + return nil + } + if a.SecurityGroupSelector != nil { + errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, securityGroupSelectorPath)) + } + if a.MetadataOptions != nil { + errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, metadataOptionsPath)) + } + if a.AMIFamily != nil { + errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, amiFamilyPath)) + } + if a.InstanceProfile != nil { + errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, instanceProfilePath)) + } return errs } func (a *AWS) validateSubnets() (errs *apis.FieldError) { if a.SubnetSelector == nil { - errs = errs.Also(apis.ErrMissingField("subnetSelector")) + errs = errs.Also(apis.ErrMissingField(fieldPathSubnetSelectorPath)) } for key, value := range a.SubnetSelector { if key == "" || value == "" { - errs = errs.Also(apis.ErrInvalidValue("\"\"", fmt.Sprintf("subnetSelector['%s']", key))) + errs = errs.Also(apis.ErrInvalidValue("\"\"", fmt.Sprintf("%s['%s']", fieldPathSubnetSelectorPath, key))) } } return errs } func (a *AWS) validateSecurityGroups() (errs *apis.FieldError) { + if a.LaunchTemplate != nil { + return nil + } if a.SecurityGroupSelector == nil { - errs = errs.Also(apis.ErrMissingField("securityGroupSelector")) + errs = errs.Also(apis.ErrMissingField(securityGroupSelectorPath)) } for key, value := range a.SecurityGroupSelector { if key == "" || value == "" { - errs = errs.Also(apis.ErrInvalidValue("\"\"", fmt.Sprintf("securityGroupSelector['%s']", key))) + errs = errs.Also(apis.ErrInvalidValue("\"\"", fmt.Sprintf("%s['%s']", securityGroupSelectorPath, key))) } } return errs @@ -87,7 +113,7 @@ func (a *AWS) validateMetadataOptions() (errs *apis.FieldError) { a.validateHTTPProtocolIpv6(), a.validateHTTPPutResponseHopLimit(), a.validateHTTPTokens(), - ).ViaField("metadataOptions") + ).ViaField(metadataOptionsPath) } func (a *AWS) validateHTTPEndpoint() (errs *apis.FieldError) { @@ -126,7 +152,7 @@ func (a *AWS) validateAMIFamily() *apis.FieldError { if a.AMIFamily == nil { return nil } - return a.validateStringEnum(*a.AMIFamily, "amiFamily", SupportedAMIFamilies) + return a.validateStringEnum(*a.AMIFamily, amiFamilyPath, SupportedAMIFamilies) } func (a *AWS) validateStringEnum(value, field string, validValues []string) *apis.FieldError { diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/register.go b/pkg/cloudprovider/aws/apis/v1alpha1/register.go index 714979c5d1e3..bff17dee9fc8 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/register.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/register.go @@ -33,10 +33,12 @@ var ( "k8s.aws", } AMIFamilyBottlerocket = "Bottlerocket" - AMIFamilyEKSOptimized = "EKSOptimized" + AMIFamilyAL2 = "AL2" + AMIFamilyUbuntu = "Ubuntu" SupportedAMIFamilies = []string{ AMIFamilyBottlerocket, - AMIFamilyEKSOptimized, + AMIFamilyAL2, + AMIFamilyUbuntu, } ) diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go index eadd0c255924..caa41b1c9853 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go @@ -33,6 +33,11 @@ func (in *AWS) DeepCopyInto(out *AWS) { *out = new(string) **out = **in } + if in.InstanceProfile != nil { + in, out := &in.InstanceProfile, &out.InstanceProfile + *out = new(string) + **out = **in + } if in.LaunchTemplate != nil { in, out := &in.LaunchTemplate, &out.LaunchTemplate *out = new(string) diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 435ec8aabaee..1b4f7166597e 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -283,7 +283,7 @@ func (p *LaunchTemplateProvider) getUserData(ctx context.Context, constraints *v if aws.StringValue(constraints.AMIFamily) == v1alpha1.AMIFamilyBottlerocket { return p.getBottlerocketUserData(ctx, constraints, additionalLabels, caBundle) } - return p.getEKSOptimizedUserData(ctx, constraints, instanceTypes, additionalLabels, caBundle) + return p.getAL2UserData(ctx, constraints, instanceTypes, additionalLabels, caBundle) } func (p *LaunchTemplateProvider) getBottlerocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string, caBundle *string) string { @@ -311,10 +311,11 @@ func (p *LaunchTemplateProvider) getBottlerocketUserData(ctx context.Context, co return base64.StdEncoding.EncodeToString([]byte(userData)) } -// getEKSOptimizedUserData returns the exact same string for equivalent input, +// getAL2UserData returns the exact same string for equivalent input, // even if elements of those inputs are in differing orders, // guaranteeing it won't cause spurious hash differences. -func (p *LaunchTemplateProvider) getEKSOptimizedUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string, caBundle *string) string { +// AL2 userdata also works on Ubuntu +func (p *LaunchTemplateProvider) getAL2UserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string, caBundle *string) string { var containerRuntimeArg string if !needsDocker(instanceTypes) { containerRuntimeArg = "--container-runtime containerd" @@ -392,8 +393,8 @@ func (p *LaunchTemplateProvider) getNodeTaintArgs(constraints *v1alpha1.Constrai } func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, constraints *v1alpha1.Constraints) (string, error) { - if constraints.InstanceProfile != "" { - return constraints.InstanceProfile, nil + if constraints.InstanceProfile != nil { + return aws.StringValue(constraints.InstanceProfile), nil } defaultProfile := injection.GetOptions(ctx).AWSDefaultInstanceProfile if defaultProfile == "" { diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index e94ca3b27f9d..258c37d01ae8 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -519,7 +519,7 @@ var _ = Describe("Allocation", func() { Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("test-instance-profile")) }) It("should use the instance profile on the Provisioner when specified", func() { - provider = &v1alpha1.AWS{InstanceProfile: "overridden-profile"} + provider = &v1alpha1.AWS{InstanceProfile: aws.String("overridden-profile")} ProvisionerWithProvider(&v1alpha5.Provisioner{ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}}, provider) provisioner.SetDefaults(ctx) @@ -568,7 +568,7 @@ var _ = Describe("Allocation", func() { provisioner.SetDefaults(ctx) constraints, err := v1alpha1.Deserialize(&provisioner.Spec.Constraints) Expect(err).ToNot(HaveOccurred()) - Expect(constraints.InstanceProfile).To(Equal("")) + Expect(constraints.InstanceProfile).To(BeNil()) }) It("should default requirements", func() { diff --git a/website/content/en/preview/AWS/provisioning.md b/website/content/en/preview/AWS/provisioning.md index 144a91f144b0..d91ef0745451 100644 --- a/website/content/en/preview/AWS/provisioning.md +++ b/website/content/en/preview/AWS/provisioning.md @@ -150,6 +150,21 @@ spec: httpTokens: required ``` +### Amazon Machine Image (AMI) Family + +The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). + +Currently, Karpenter supports `amiFamily` values `al2`, `bottlerocket`, and `ubuntu`. GPUs are only supported with `al2`. + +Note: If a custom launch template is specified, then the AMI value in the launch template is used rather than the `amiFamily` value. + + +``` +spec: + provider: + amiFamily: bottlerocket +``` + ## Other Resources diff --git a/website/content/en/preview/development-guide.md b/website/content/en/preview/development-guide.md index 00b9e6426b0d..6bf1fb9f9cdf 100644 --- a/website/content/en/preview/development-guide.md +++ b/website/content/en/preview/development-guide.md @@ -49,7 +49,7 @@ make dev # run codegen, lint, and tests ### Testing ```bash -make test # E2e correctness tests +make test # E2E correctness tests make battletest # More rigorous tests run in CI environment ```