From 2f76ddbabfd732abd35ca0ce30777fe00c06bdb0 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Mon, 29 Jan 2024 10:11:40 -0800 Subject: [PATCH] feat: al2023 AMI family --- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 1 + pkg/apis/v1beta1/ec2nodeclass.go | 2 +- pkg/apis/v1beta1/labels.go | 1 + pkg/operator/operator.go | 28 +++ pkg/operator/options/options.go | 2 + pkg/providers/amifamily/al2023.go | 62 ++++++ .../amifamily/bootstrap/bootstrap.go | 1 + pkg/providers/amifamily/bootstrap/nodeadm.go | 178 ++++++++++++++++++ pkg/providers/amifamily/resolver.go | 3 + .../launchtemplate/launchtemplate.go | 1 + 10 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 pkg/providers/amifamily/al2023.go create mode 100644 pkg/providers/amifamily/bootstrap/nodeadm.go diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index af4acba6004c..456eb9b47a8d 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -50,6 +50,7 @@ spec: description: AMIFamily is the AMI family that instances use. enum: - AL2 + - AL2023 - Bottlerocket - Ubuntu - Custom diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 17b607278581..baa84b0317d9 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -48,7 +48,7 @@ type EC2NodeClassSpec struct { // +optional AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"` // AMIFamily is the AMI family that instances use. - // +kubebuilder:validation:Enum:={AL2,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} // +required AMIFamily *string `json:"amiFamily"` // UserData to be applied to the provisioned nodes. diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index 1b86eb70726f..a938aa2d48cf 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -73,6 +73,7 @@ var ( } AMIFamilyBottlerocket = "Bottlerocket" AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" AMIFamilyUbuntu = "Ubuntu" AMIFamilyWindows2019 = "Windows2019" AMIFamilyWindows2022 = "Windows2022" diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 33d6d4528c84..2e999fd720ab 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -122,6 +122,13 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } else { logging.FromContext(ctx).With("cluster-endpoint", clusterEndpoint).Debugf("discovered cluster endpoint") } + if clusterCIDR, err := ResolveClusterCIDR(ctx, eks.New(sess)); err != nil { + logging.FromContext(ctx).Fatalf("unable to detect the cluster CIDR, %s", err) + } else { + logging.FromContext(ctx).With("cluster-cidr", clusterCIDR).Debugf("discovered cluster CIDR") + // Inject clusterCIDR back into options for use w/ the AL2023 AMI + options.FromContext(ctx).ClusterCIDR = clusterCIDR + } // We perform best-effort on resolving the kube-dns IP kubeDNSIP, err := kubeDNSIP(ctx, operator.KubernetesInterface) if err != nil { @@ -234,6 +241,27 @@ func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string return *out.Cluster.Endpoint, nil } +func ResolveClusterCIDR(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { + if cidr := options.FromContext(ctx).ClusterCIDR; cidr != "" { + return cidr, nil + } + out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ + Name: aws.String(options.FromContext(ctx).ClusterName), + }) + if err != nil { + return "", fmt.Errorf("failed to resolve cluster CIDR, %w", err) + } + + if ipv4CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr; ipv4CIDR != nil { + return *ipv4CIDR, nil + } + if ipv6CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr; ipv6CIDR != nil { + return *ipv6CIDR, nil + } + + return "", fmt.Errorf("failed to resolve cluster CIDR") +} + func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) { // Discover CA Bundle from the REST client. We could alternatively // have used the simpler client-go InClusterConfig() method. diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 0aad85f970d6..07e4dd1dc1c4 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -38,6 +38,7 @@ type Options struct { ClusterCABundle string ClusterName string ClusterEndpoint string + ClusterCIDR string IsolatedVPC bool VMMemoryOverheadPercent float64 InterruptionQueue string @@ -50,6 +51,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") + fs.StringVar(&o.ClusterCIDR, "cluster-cidr", env.WithDefaultString("CLUSTER_CIDR", ""), "The IP address range from which cluster services will receive IP addresses. Corresponds to cluster service IPv4 / IPv6 range.") fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") diff --git a/pkg/providers/amifamily/al2023.go b/pkg/providers/amifamily/al2023.go new file mode 100644 index 000000000000..e9b4658b0fe3 --- /dev/null +++ b/pkg/providers/amifamily/al2023.go @@ -0,0 +1,62 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" +) + +type AL2023 struct { + DefaultFamily + *Options +} + +func (a AL2023) DefaultAMIs(version string) []DefaultAMIOutput { + // TODO: SSM parameters not yet available + return []DefaultAMIOutput{} +} + +func (a AL2023) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, _ *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { + return bootstrap.Nodeadm{ + Options: bootstrap.Options{ + ClusterName: a.Options.ClusterName, + ClusterEndpoint: a.Options.ClusterEndpoint, + ClusterCIDR: a.Options.ClusterCIDR, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + CustomUserData: customUserData, + }, + } +} + +// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family +func (a AL2023) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { + return []*v1beta1.BlockDeviceMapping{{ + DeviceName: a.EphemeralBlockDevice(), + EBS: &DefaultEBS, + }} +} + +func (a AL2023) EphemeralBlockDevice() *string { + return lo.ToPtr("/dev/xvda") +} diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go index 65321c40d339..fb825ed42e02 100644 --- a/pkg/providers/amifamily/bootstrap/bootstrap.go +++ b/pkg/providers/amifamily/bootstrap/bootstrap.go @@ -34,6 +34,7 @@ import ( type Options struct { ClusterName string ClusterEndpoint string + ClusterCIDR string KubeletConfig *corev1beta1.KubeletConfiguration Taints []core.Taint `hash:"set"` Labels map[string]string `hash:"set"` diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go new file mode 100644 index 000000000000..f0a927b7557f --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/nodeadm.go @@ -0,0 +1,178 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "bytes" + "encoding/base64" + "fmt" + "mime/multipart" + "net/textproto" + "reflect" + "strings" + + admapi "github.com/awslabs/amazon-eks-ami/nodeadm/api" + "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1" + "github.com/samber/lo" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" +) + +const nodeConfigContentType = "application/" + admapi.GroupName + +// const shellConfigContentType = `text/x-shellscript; charset="us-ascii"` + +type Nodeadm struct { + Options +} + +func (n Nodeadm) Script() (string, error) { + nodeadmConfig, err := n.nodeadmConfig() + if err != nil { + return "", fmt.Errorf("generating NodeConfig, %w", err) + } + userData, err := n.mergeUserData(lo.Compact([]string{nodeadmConfig, lo.FromPtr(n.CustomUserData)})...) + if err != nil { + return "", err + } + // The mime/multipart package adds carriage returns, while the rest of our logic does not. Remove all + // carriage returns for consistency. + return base64.StdEncoding.EncodeToString([]byte(strings.ReplaceAll(userData, "\r", ""))), nil +} + +func (n Nodeadm) mergeUserData(userDatas ...string) (string, error) { + var outputBuffer bytes.Buffer + writer := multipart.NewWriter(&outputBuffer) + if err := writer.SetBoundary(Boundary); err != nil { + return "", fmt.Errorf("defining boundary for merged user data %w", err) + } + outputBuffer.WriteString(MIMEVersionHeader + "\n") + outputBuffer.WriteString(fmt.Sprintf(MIMEContentTypeHeaderTemplate, Boundary) + "\n\n") + for _, userData := range userDatas { + mimedUserData, err := n.mimeify(userData) + if err != nil { + return "", err + } + if err := copyCustomUserDataParts(writer, mimedUserData); err != nil { + return "", err + } + } + writer.Close() + return outputBuffer.String(), nil +} + +func (n Nodeadm) nodeadmConfig() (string, error) { + config := &v1alpha1.NodeConfig{ + TypeMeta: v1.TypeMeta{ + Kind: "NodeConfig", + APIVersion: admapi.GroupName + "/v1alpha1", + }, + Spec: v1alpha1.NodeConfigSpec{ + Cluster: v1alpha1.ClusterDetails{ + Name: n.ClusterName, + APIServerEndpoint: n.ClusterEndpoint, + CIDR: n.ClusterCIDR, + }, + }, + } + if n.CABundle != nil { + ca, err := base64.StdEncoding.DecodeString(*n.CABundle) + if err != nil { + return "", err + } + config.Spec.Cluster.CertificateAuthority = ca + } + inlineConfig, err := n.generateInlineKubeletConfiguration() + if err != nil { + return "", err + } + if len(inlineConfig) != 0 { + config.Spec.Kubelet.Config = inlineConfig + } + if labelArg := n.nodeLabelArg(); labelArg != "" { + config.Spec.Kubelet.Flags = []string{labelArg} + } + + configJSON, err := json.Marshal(config) + if err != nil { + return "", err + } + return string(configJSON), nil +} + +func (n Nodeadm) generateInlineKubeletConfiguration() (map[string]runtime.RawExtension, error) { + config := map[string]runtime.RawExtension{} + if n.KubeletConfig != nil { + t := reflect.TypeOf(n.KubeletConfig).Elem() + rv := reflect.Indirect(reflect.ValueOf(n.KubeletConfig)) + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + name := "" + if tags := strings.Split(field.Tag.Get("json"), ","); len(tags) > 0 { + name = tags[0] + } else { + return nil, fmt.Errorf("failed to serialize KubeletConfiguration, field %q doesn't specify a json tag", field.Name) + } + val := rv.FieldByName(field.Name).Interface() + if reflect.DeepEqual(val, reflect.Zero(field.Type).Interface()) { + // don't attempt to serialize zero values + continue + } + jsonVal, err := json.Marshal(val) + if err != nil { + return nil, fmt.Errorf("failed to serialize KubeletConfiguration, %w", err) + } + config[name] = runtime.RawExtension{Raw: jsonVal} + } + } + if len(n.Taints) != 0 { + taintsJSON, err := json.Marshal(n.Taints) + if err != nil { + return nil, fmt.Errorf("failed to serialize KubeletConfiguration, %w", err) + } + config["registerWithTaints"] = runtime.RawExtension{Raw: taintsJSON} + } + if _, ok := config["maxPods"]; !n.AWSENILimitedPodDensity && !ok { + config["maxPods"] = runtime.RawExtension{Raw: []byte("110")} + } + return config, nil +} + +// TODO: attempt to parse as yaml / json then fall back to shell-script? Or only support yaml / json in non-mime format +// mimeify returns userData in a mime format +// if the userData passed in is already in a mime format, then the input is returned without modification +func (n Nodeadm) mimeify(customUserData string) (string, error) { + if strings.HasPrefix(strings.TrimSpace(customUserData), "MIME-Version:") || + strings.HasPrefix(strings.TrimSpace(customUserData), "Content-Type:") { + return customUserData, nil + } + var outputBuffer bytes.Buffer + writer := multipart.NewWriter(&outputBuffer) + outputBuffer.WriteString(MIMEVersionHeader + "\n") + outputBuffer.WriteString(fmt.Sprintf(MIMEContentTypeHeaderTemplate, writer.Boundary()) + "\n\n") + partWriter, err := writer.CreatePart(textproto.MIMEHeader{ + "Content-Type": []string{nodeConfigContentType}, + }) + if err != nil { + return "", fmt.Errorf("creating multi-part section from custom user-data: %w", err) + } + _, err = partWriter.Write([]byte(customUserData)) + if err != nil { + return "", fmt.Errorf("writing custom user-data input: %w", err) + } + writer.Close() + return outputBuffer.String(), nil +} diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 14467a6d852a..2c6fbbccc676 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -50,6 +50,7 @@ type Resolver struct { type Options struct { ClusterName string ClusterEndpoint string + ClusterCIDR string InstanceProfile string CABundle *string `hash:"ignore"` InstanceStorePolicy *v1beta1.InstanceStorePolicy @@ -175,6 +176,8 @@ func GetAMIFamily(amiFamily *string, options *Options) AMIFamily { return &Windows{Options: options, Version: v1beta1.Windows2022, Build: v1beta1.Windows2022Build} case v1beta1.AMIFamilyCustom: return &Custom{Options: options} + case v1beta1.AMIFamilyAL2023: + return &AL2023{Options: options} default: return &AL2{Options: options} } diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index ea01721b0dad..b7d83b6e88bb 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -172,6 +172,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N options := &amifamily.Options{ ClusterName: options.FromContext(ctx).ClusterName, ClusterEndpoint: p.ClusterEndpoint, + ClusterCIDR: options.FromContext(ctx).ClusterCIDR, InstanceProfile: instanceProfile, InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, SecurityGroups: lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) v1beta1.SecurityGroup {