From c1d08c6c91019443746ef310f2cd778335b4be65 Mon Sep 17 00:00:00 2001 From: Brandon Wagner Date: Thu, 10 Mar 2022 13:40:49 -0600 Subject: [PATCH] Add BlockDeviceMappings to the AWS cloudprovider (#1420) * Add support for specifying blockDeviceMappings in the provisioner * cleanup * a few fixes * rename pkg ltresolver to amifamily * naming changes --- go.mod | 1 + go.sum | 6 +- pkg/cloudprovider/aws/ami.go | 139 -------- pkg/cloudprovider/aws/amifamily/al2.go | 68 ++++ pkg/cloudprovider/aws/amifamily/ami.go | 48 +++ .../aws/amifamily/bootstrap/bootstrap.go | 40 +++ .../aws/amifamily/bootstrap/bottlerocket.go | 73 ++++ .../aws/amifamily/bootstrap/eksbootstrap.go | 79 +++++ .../aws/amifamily/bottlerocket.go | 76 ++++ pkg/cloudprovider/aws/amifamily/resolver.go | 137 ++++++++ pkg/cloudprovider/aws/amifamily/ubuntu.go | 59 ++++ .../aws/apis/v1alpha1/provider.go | 101 ++++-- .../aws/apis/v1alpha1/provider_defaults.go | 11 - .../aws/apis/v1alpha1/provider_validation.go | 70 +++- .../apis/v1alpha1/zz_generated.deepcopy.go | 122 ++++++- pkg/cloudprovider/aws/cloudprovider.go | 71 ++-- pkg/cloudprovider/aws/instance.go | 25 +- pkg/cloudprovider/aws/launchtemplate.go | 324 ++++-------------- pkg/cloudprovider/aws/securitygroups.go | 18 +- pkg/cloudprovider/aws/subnets.go | 8 +- pkg/cloudprovider/aws/suite_test.go | 190 +++++++++- pkg/controllers/metrics/pod/suite_test.go | 2 +- pkg/controllers/provisioning/controller.go | 4 + .../provisioning/scheduling/suite_test.go | 8 +- pkg/controllers/provisioning/suite_test.go | 1 - pkg/controllers/selection/suite_test.go | 1 - .../content/en/preview/AWS/provisioning.md | 21 ++ 27 files changed, 1200 insertions(+), 503 deletions(-) delete mode 100644 pkg/cloudprovider/aws/ami.go create mode 100644 pkg/cloudprovider/aws/amifamily/al2.go create mode 100644 pkg/cloudprovider/aws/amifamily/ami.go create mode 100644 pkg/cloudprovider/aws/amifamily/bootstrap/bootstrap.go create mode 100644 pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go create mode 100644 pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go create mode 100644 pkg/cloudprovider/aws/amifamily/bottlerocket.go create mode 100644 pkg/cloudprovider/aws/amifamily/resolver.go create mode 100644 pkg/cloudprovider/aws/amifamily/ubuntu.go diff --git a/go.mod b/go.mod index 1f76715abd1e..4af1363827a1 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.18.1 github.com/patrickmn/go-cache v2.1.0+incompatible + github.com/pelletier/go-toml/v2 v2.0.0-beta.5 github.com/prometheus/client_golang v1.12.1 github.com/prometheus/client_model v0.2.0 go.uber.org/multierr v1.7.0 diff --git a/go.sum b/go.sum index 346d608c16c7..31647347f9d8 100644 --- a/go.sum +++ b/go.sum @@ -593,7 +593,10 @@ github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FI github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml/v2 v2.0.0-beta.5 h1:zRY68WYiDE9YZLflUWuyOny5YA+DwvyFzZBfb8E/9wk= +github.com/pelletier/go-toml/v2 v2.0.0-beta.5/go.mod h1:ke6xncR3W76Ba8xnVxkrZG0js6Rd2BsQEAYrfgJ6eQA= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -687,8 +690,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942 h1:t0lM6y/M5IiUZyvbBTcngso8SZEZICH7is9B6g/obVU= +github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/pkg/cloudprovider/aws/ami.go b/pkg/cloudprovider/aws/ami.go deleted file mode 100644 index 0ba2dbbf666c..000000000000 --- a/pkg/cloudprovider/aws/ami.go +++ /dev/null @@ -1,139 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "context" - "fmt" - "strings" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ssm" - "github.com/aws/aws-sdk-go/service/ssm/ssmiface" - "github.com/patrickmn/go-cache" - "k8s.io/client-go/kubernetes" - "knative.dev/pkg/logging" - - "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" - "github.com/aws/karpenter/pkg/cloudprovider" - "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" -) - -const kubernetesVersionCacheKey = "kubernetesVersion" - -type AMIProvider struct { - cache *cache.Cache - ssm ssmiface.SSMAPI - clientSet *kubernetes.Clientset -} - -func NewAMIProvider(ssm ssmiface.SSMAPI, clientSet *kubernetes.Clientset) *AMIProvider { - return &AMIProvider{ - ssm: ssm, - clientSet: clientSet, - cache: cache.New(CacheTTL, CacheCleanupInterval), - } -} - -// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc -func (p *AMIProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType) (map[string][]cloudprovider.InstanceType, error) { - version, err := p.kubeServerVersion(ctx) - if err != nil { - return nil, fmt.Errorf("kube server version, %w", err) - } - // Separate instance types by unique queries - amiQueries := map[string][]cloudprovider.InstanceType{} - for _, instanceType := range instanceTypes { - query := p.getSSMQuery(constraints, instanceType, version) - amiQueries[query] = append(amiQueries[query], instanceType) - } - // Separate instance types by unique AMIIDs - amiIDs := map[string][]cloudprovider.InstanceType{} - for query, instanceTypes := range amiQueries { - amiID, err := p.getAMIID(ctx, query) - if err != nil { - return nil, err - } - amiIDs[amiID] = instanceTypes - } - return amiIDs, nil -} - -func (p *AMIProvider) getAMIID(ctx context.Context, query string) (string, error) { - if id, ok := p.cache.Get(query); ok { - return id.(string), nil - } - output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(query)}) - if err != nil { - return "", fmt.Errorf("getting ssm parameter, %w", err) - } - ami := aws.StringValue(output.Parameter.Value) - p.cache.SetDefault(query, ami) - logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, query) - return ami, nil -} - -func (p *AMIProvider) getSSMQuery(constraints *v1alpha1.Constraints, instanceType cloudprovider.InstanceType, version string) string { - switch aws.StringValue(constraints.AMIFamily) { - case v1alpha1.AMIFamilyBottlerocket: - return p.getBottlerocketAlias(version, instanceType) - case v1alpha1.AMIFamilyUbuntu: - return p.getUbuntuAlias(version, instanceType) - } - return p.getAL2Alias(version, instanceType) -} - -// getAL2Alias returns a properly-formatted alias for an Amazon Linux AMI from SSM -func (p *AMIProvider) getAL2Alias(version string, instanceType cloudprovider.InstanceType) string { - amiSuffix := "" - if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { - amiSuffix = "-gpu" - } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { - amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture()) - } - return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) -} - -// getBottlerocketAlias returns a properly-formatted alias for a Bottlerocket AMI from SSM -func (p *AMIProvider) getBottlerocketAlias(version string, instanceType cloudprovider.InstanceType) string { - arch := "x86_64" - amiSuffix := "" - if !instanceType.NvidiaGPUs().IsZero() { - amiSuffix = "-nvidia" - } - if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { - arch = instanceType.Architecture() - } - return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s%s/%s/latest/image_id", version, amiSuffix, arch) -} - -// getUbuntuAlias returns a properly-formatted alias for an Ubuntu AMI from SSM -func (p *AMIProvider) getUbuntuAlias(version string, instanceType cloudprovider.InstanceType) string { - return fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, instanceType.Architecture()) -} - -func (p *AMIProvider) kubeServerVersion(ctx context.Context) (string, error) { - if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { - return version.(string), nil - } - serverVersion, err := p.clientSet.Discovery().ServerVersion() - if err != nil { - return "", err - } - version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+")) - p.cache.SetDefault(kubernetesVersionCacheKey, version) - logging.FromContext(ctx).Debugf("Discovered kubernetes version %s", version) - return version, nil -} diff --git a/pkg/cloudprovider/aws/amifamily/al2.go b/pkg/cloudprovider/aws/amifamily/al2.go new file mode 100644 index 000000000000..7a391c828356 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/al2.go @@ -0,0 +1,68 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + core "k8s.io/api/core/v1" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" +) + +type AL2 struct { + *Options +} + +// SSMAlias returns the AMI Alias to query SSM +func (a AL2) SSMAlias(version string, instanceType cloudprovider.InstanceType) string { + amiSuffix := "" + if !instanceType.NvidiaGPUs().IsZero() || !instanceType.AWSNeurons().IsZero() { + amiSuffix = "-gpu" + } else if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { + amiSuffix = fmt.Sprintf("-%s", instanceType.Architecture()) + } + return fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2%s/recommended/image_id", version, amiSuffix) +} + +// UserData returns the exact same string for equivalent input, +// even if elements of those inputs are in differing orders, +// guaranteeing it won't cause spurious hash differences. +// AL2 userdata also works on Ubuntu +func (a AL2) UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper { + return bootstrap.EKS{ + Options: bootstrap.Options{ + ClusterName: a.Options.ClusterName, + ClusterEndpoint: a.Options.ClusterEndpoint, + AWSENILimitedPodDensity: a.Options.AWSENILimitedPodDensity, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + }, + } +} + +// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family +func (a AL2) DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping { + return []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &defaultEBS, + }} +} diff --git a/pkg/cloudprovider/aws/amifamily/ami.go b/pkg/cloudprovider/aws/amifamily/ami.go new file mode 100644 index 000000000000..5b1451a67315 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/ami.go @@ -0,0 +1,48 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/patrickmn/go-cache" + "knative.dev/pkg/logging" + + "github.com/aws/karpenter/pkg/cloudprovider" +) + +type AMIProvider struct { + cache *cache.Cache + ssm ssmiface.SSMAPI +} + +// Get returns a set of AMIIDs and corresponding instance types. AMI may vary due to architecture, accelerator, etc +func (p *AMIProvider) Get(ctx context.Context, instanceType cloudprovider.InstanceType, ssmQuery string) (string, error) { + if id, ok := p.cache.Get(ssmQuery); ok { + return id.(string), nil + } + output, err := p.ssm.GetParameterWithContext(ctx, &ssm.GetParameterInput{Name: aws.String(ssmQuery)}) + if err != nil { + return "", fmt.Errorf("getting ssm parameter, %w", err) + } + ami := aws.StringValue(output.Parameter.Value) + p.cache.SetDefault(ssmQuery, ami) + logging.FromContext(ctx).Debugf("Discovered %s for query %s", ami, ssmQuery) + return ami, nil +} diff --git a/pkg/cloudprovider/aws/amifamily/bootstrap/bootstrap.go b/pkg/cloudprovider/aws/amifamily/bootstrap/bootstrap.go new file mode 100644 index 000000000000..b41accfb4240 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/bootstrap/bootstrap.go @@ -0,0 +1,40 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + core "k8s.io/api/core/v1" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" +) + +// Options is the node bootstrapping parameters passed from Karpenter to the provisioning node +type Options struct { + ClusterName string + ClusterEndpoint string + KubeletConfig v1alpha5.KubeletConfiguration + Taints []core.Taint `hash:"set"` + Labels map[string]string `hash:"set"` + CABundle *string + AWSENILimitedPodDensity bool +} + +// Bootstrapper can be implemented to generate a bootstrap script +// that uses the params from the Bootstrap type for a specific +// bootstrapping method. +// Examples are the Bottlerocket config and the eks-bootstrap script +type Bootstrapper interface { + Script() string +} diff --git a/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go new file mode 100644 index 000000000000..7fb24f9ddc6d --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/bootstrap/bottlerocket.go @@ -0,0 +1,73 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "encoding/base64" + "fmt" + + "github.com/pelletier/go-toml/v2" +) + +type Bottlerocket struct { + Options +} + +// config is the root of the bottlerocket config, see more here https://github.com/bottlerocket-os/bottlerocket#using-user-data +type config struct { + Settings settings `toml:"settings"` +} + +// settings is part of the bottlerocket config +type settings struct { + Kubernetes kubernetes `toml:"kubernetes"` +} + +// kubernetes specific configuration for bottlerocket api +type kubernetes struct { + APIServer string `toml:"api-server"` + ClusterCertificate *string `toml:"cluster-certificate"` + ClusterName string `toml:"cluster-name,omitempty"` + ClusterDNSIP string `toml:"cluster-dns-ip,omitempty"` + NodeLabels map[string]string `toml:"node-labels,omitempty"` + NodeTaints map[string][]string `toml:"node-taints,omitempty"` + MaxPods int `toml:"max-pods,omitempty"` +} + +func (b Bottlerocket) Script() string { + s := config{Settings: settings{ + Kubernetes: kubernetes{ + ClusterName: b.ClusterName, + APIServer: b.ClusterEndpoint, + ClusterCertificate: b.CABundle, + NodeLabels: b.Labels, + }, + }} + if len(b.KubeletConfig.ClusterDNS) > 0 { + s.Settings.Kubernetes.ClusterDNSIP = b.KubeletConfig.ClusterDNS[0] + } + if !b.AWSENILimitedPodDensity { + s.Settings.Kubernetes.MaxPods = 110 + } + s.Settings.Kubernetes.NodeTaints = map[string][]string{} + for _, taint := range b.Taints { + s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s:%s", taint.Value, taint.Effect)) + } + script, err := toml.Marshal(s) + if err != nil { + panic(err) + } + return base64.StdEncoding.EncodeToString(script) +} diff --git a/pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go b/pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go new file mode 100644 index 000000000000..bf833f106b7c --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/bootstrap/eksbootstrap.go @@ -0,0 +1,79 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + "sync" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" +) + +type EKS struct { + Options +} + +func (e EKS) Script() string { + var caBundleArg string + if e.CABundle != nil { + caBundleArg = fmt.Sprintf("--b64-cluster-ca='%s'", *e.CABundle) + } + var userData bytes.Buffer + userData.WriteString("#!/bin/bash -xe\n") + userData.WriteString("exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1\n") + userData.WriteString(fmt.Sprintf("/etc/eks/bootstrap.sh '%s' --apiserver-endpoint='%s' %s", e.ClusterName, e.ClusterEndpoint, caBundleArg)) + + kubeletExtraArgs := strings.Join([]string{e.nodeLabelArg(), e.nodeTaintArg()}, " ") + + if !e.AWSENILimitedPodDensity { + userData.WriteString(" \\\n--use-max-pods=false") + kubeletExtraArgs += " --max-pods=110" + } + if kubeletExtraArgs = strings.Trim(kubeletExtraArgs, " "); len(kubeletExtraArgs) > 0 { + userData.WriteString(fmt.Sprintf(" \\\n--kubelet-extra-args='%s'", kubeletExtraArgs)) + } + if len(e.KubeletConfig.ClusterDNS) > 0 { + userData.WriteString(fmt.Sprintf(" \\\n--dns-cluster-ip='%s'", e.KubeletConfig.ClusterDNS[0])) + } + return base64.StdEncoding.EncodeToString(userData.Bytes()) +} + +func (e EKS) nodeTaintArg() string { + nodeTaintsArg := "" + taintStrings := []string{} + var once sync.Once + for _, taint := range e.Taints { + once.Do(func() { nodeTaintsArg = "--register-with-taints=" }) + taintStrings = append(taintStrings, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) + } + return fmt.Sprintf("%s%s", nodeTaintsArg, strings.Join(taintStrings, ",")) +} + +func (e EKS) nodeLabelArg() string { + nodeLabelArg := "" + labelStrings := []string{} + var once sync.Once + for k, v := range e.Labels { + if v1alpha5.AllowedLabelDomains.Has(k) { + continue + } + once.Do(func() { nodeLabelArg = "--node-labels=" }) + labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", k, v)) + } + return fmt.Sprintf("%s%s", nodeLabelArg, strings.Join(labelStrings, ",")) +} diff --git a/pkg/cloudprovider/aws/amifamily/bottlerocket.go b/pkg/cloudprovider/aws/amifamily/bottlerocket.go new file mode 100644 index 000000000000..c9cb6b801aa8 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/bottlerocket.go @@ -0,0 +1,76 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" +) + +type Bottlerocket struct { + *Options +} + +// SSMAlias returns the AMI Alias to query SSM +func (b Bottlerocket) SSMAlias(version string, instanceType cloudprovider.InstanceType) string { + arch := "x86_64" + amiSuffix := "" + if !instanceType.NvidiaGPUs().IsZero() { + amiSuffix = "-nvidia" + } + if instanceType.Architecture() == v1alpha5.ArchitectureArm64 { + arch = instanceType.Architecture() + } + return fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s%s/%s/latest/image_id", version, amiSuffix, arch) +} + +// UserData returns the default userdata script for the AMI Family +func (b Bottlerocket) UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper { + return bootstrap.Bottlerocket{ + Options: bootstrap.Options{ + ClusterName: b.Options.ClusterName, + ClusterEndpoint: b.Options.ClusterEndpoint, + AWSENILimitedPodDensity: b.Options.AWSENILimitedPodDensity, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + }, + } +} + +// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family +func (b Bottlerocket) DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping { + xvdaEBS := defaultEBS + xvdaEBS.VolumeSize = resource.NewScaledQuantity(4, resource.Giga) + return []*v1alpha1.BlockDeviceMapping{ + { + DeviceName: aws.String("/dev/xvda"), + EBS: &xvdaEBS, + }, + { + DeviceName: aws.String("/dev/xvdb"), + EBS: &defaultEBS, + }, + } +} diff --git a/pkg/cloudprovider/aws/amifamily/resolver.go b/pkg/cloudprovider/aws/amifamily/resolver.go new file mode 100644 index 000000000000..ff603009c891 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/resolver.go @@ -0,0 +1,137 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "context" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ssm/ssmiface" + "github.com/patrickmn/go-cache" + core "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" +) + +var defaultEBS = v1alpha1.BlockDevice{ + Encrypted: aws.Bool(true), + VolumeType: aws.String(ec2.VolumeTypeGp3), + VolumeSize: resource.NewScaledQuantity(20, resource.Giga), +} + +// Resolver is able to fill-in dynamic launch template parameters +type Resolver struct { + amiProvider *AMIProvider +} + +// Options define the static launch template parameters +type Options struct { + ClusterName string + ClusterEndpoint string + AWSENILimitedPodDensity bool + InstanceProfile string + CABundle *string `hash:"ignore"` + // Level-triggered fields that may change out of sync. + KubernetesVersion string + SecurityGroupsIDs []string + Tags map[string]string + Labels map[string]string `hash:"ignore"` +} + +// LaunchTemplate holds the dynamically generated launch template parameters +type LaunchTemplate struct { + *Options + UserData bootstrap.Bootstrapper + BlockDeviceMappings []*v1alpha1.BlockDeviceMapping + MetadataOptions *v1alpha1.MetadataOptions + AMIID string + InstanceTypes []cloudprovider.InstanceType `hash:"ignore"` +} + +// AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters +type AMIFamily interface { + UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper + SSMAlias(version string, instanceType cloudprovider.InstanceType) string + DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping + DefaultMetadataOptions() *v1alpha1.MetadataOptions +} + +// New constructs a new launch template Resolver +func New(ssm ssmiface.SSMAPI, c *cache.Cache) *Resolver { + return &Resolver{ + amiProvider: &AMIProvider{ + ssm: ssm, + cache: c, + }, + } +} + +// Resolve generates launch templates using the static options and dynamically generates launch template parameters. +// Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. +func (r Resolver) Resolve(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, options *Options) ([]*LaunchTemplate, error) { + amiFamily := r.getAMIFamily(constraints.AMIFamily, options) + amiIDs := map[string][]cloudprovider.InstanceType{} + for _, instanceType := range instanceTypes { + amiID, err := r.amiProvider.Get(ctx, instanceType, amiFamily.SSMAlias(options.KubernetesVersion, instanceType)) + if err != nil { + return nil, err + } + amiIDs[amiID] = append(amiIDs[amiID], instanceType) + } + var resolvedTemplates []*LaunchTemplate + for amiID, instanceTypes := range amiIDs { + resolved := &LaunchTemplate{ + Options: options, + UserData: amiFamily.UserData(constraints.KubeletConfiguration, constraints.Taints, options.Labels, options.CABundle), + BlockDeviceMappings: constraints.BlockDeviceMappings, + MetadataOptions: constraints.MetadataOptions, + AMIID: amiID, + InstanceTypes: instanceTypes, + } + if resolved.BlockDeviceMappings == nil { + resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() + } + if resolved.MetadataOptions == nil { + resolved.MetadataOptions = amiFamily.DefaultMetadataOptions() + } + resolvedTemplates = append(resolvedTemplates, resolved) + } + return resolvedTemplates, nil +} + +func (r Resolver) getAMIFamily(amiFamily *string, options *Options) AMIFamily { + switch aws.StringValue(amiFamily) { + case v1alpha1.AMIFamilyBottlerocket: + return &Bottlerocket{Options: options} + case v1alpha1.AMIFamilyUbuntu: + return &Ubuntu{Options: options} + default: + return &AL2{Options: options} + } +} + +func (Options) DefaultMetadataOptions() *v1alpha1.MetadataOptions { + return &v1alpha1.MetadataOptions{ + HTTPEndpoint: aws.String(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled), + HTTPProtocolIPv6: aws.String(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled), + HTTPPutResponseHopLimit: aws.Int64(2), + HTTPTokens: aws.String(ec2.LaunchTemplateHttpTokensStateRequired), + } +} diff --git a/pkg/cloudprovider/aws/amifamily/ubuntu.go b/pkg/cloudprovider/aws/amifamily/ubuntu.go new file mode 100644 index 000000000000..6ea816bd4181 --- /dev/null +++ b/pkg/cloudprovider/aws/amifamily/ubuntu.go @@ -0,0 +1,59 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + core "k8s.io/api/core/v1" + + "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily/bootstrap" + "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" +) + +type Ubuntu struct { + *Options +} + +// SSMAlias returns the AMI Alias to query SSM +func (u Ubuntu) SSMAlias(version string, instanceType cloudprovider.InstanceType) string { + return fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/%s/hvm/ebs-gp2/ami-id", version, instanceType.Architecture()) +} + +// UserData returns the default userdata script for the AMI Family +func (u Ubuntu) UserData(kubeletConfig v1alpha5.KubeletConfiguration, taints []core.Taint, labels map[string]string, caBundle *string) bootstrap.Bootstrapper { + return bootstrap.EKS{ + Options: bootstrap.Options{ + ClusterName: u.Options.ClusterName, + ClusterEndpoint: u.Options.ClusterEndpoint, + AWSENILimitedPodDensity: u.Options.AWSENILimitedPodDensity, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + }, + } +} + +// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family +func (u Ubuntu) DefaultBlockDeviceMappings() []*v1alpha1.BlockDeviceMapping { + return []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/sda1"), + EBS: &defaultEBS, + }} +} diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go index 54f26354d08a..2d9101eeab22 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider.go @@ -18,20 +18,12 @@ import ( "encoding/json" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" ) -const ( - DefaultMetadataOptionsHTTPEndpoint = ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled - DefaultMetadataOptionsHTTPProtocolIPv6 = ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled - DefaultMetadataOptionsHTTPPutResponseHopLimit = 2 - DefaultMetadataOptionsHTTPTokens = ec2.LaunchTemplateHttpTokensStateRequired -) - // Constraints wraps generic constraints with AWS specific parameters type Constraints struct { *v1alpha5.Constraints @@ -50,9 +42,6 @@ type AWS struct { // InstanceProfile is the AWS identity that instances use. // +optional InstanceProfile *string `json:"instanceProfile,omitempty"` - // LaunchTemplate for the node. If not specified, a launch template will be generated. - // +optional - LaunchTemplate *string `json:"launchTemplate,omitempty"` // SubnetSelector discovers subnets by tags. A value of "" is a wildcard. // +optional SubnetSelector map[string]string `json:"subnetSelector,omitempty"` @@ -62,6 +51,16 @@ type AWS struct { // Tags to be applied on ec2 resources like instances and launch templates. // +optional Tags map[string]string `json:"tags,omitempty"` + // LaunchTemplate parameters to use when generating an LT + LaunchTemplate `json:",inline,omitempty"` +} + +type LaunchTemplate struct { + // LaunchTemplateName for the node. If not specified, a launch template will be generated. + // NOTE: This field is for specifying a custom launch template and is exposed in the Spec + // as `launchTemplate` for backwards compatibility. + // +optional + LaunchTemplateName *string `json:"launchTemplate,omitempty"` // MetadataOptions for the generated launch template of provisioned nodes. // // This specifies the exposure of the Instance Metadata Service to @@ -78,6 +77,9 @@ type AWS struct { // required. // +optional MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +optionals + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` } // MetadataOptions contains parameters for specifying the exposure of the @@ -124,6 +126,69 @@ type MetadataOptions struct { HTTPTokens *string `json:"httpTokens,omitempty"` } +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + EBS *BlockDevice `json:"ebs,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + Encrypted *bool `json:"encrypted,omitempty"` + + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + IOPS *int64 `json:"iops,omitempty"` + + // KMSKeyID (ARN) of the symmetric Key Management Service (KMS) CMK used for encryption. + KMSKeyID *string `json:"kmsKeyID,omitempty"` + + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + Throughput *int64 `json:"throughput,omitempty"` + + // VolumeSize in GiBs. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + VolumeSize *resource.Quantity `json:"volumeSize,omitempty"` + + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + VolumeType *string `json:"volumeType,omitempty"` +} + func Deserialize(constraints *v1alpha5.Constraints) (*Constraints, error) { if constraints.Provider == nil { return nil, fmt.Errorf("invariant violated: spec.provider is not defined. Is the defaulting webhook installed?") @@ -150,15 +215,3 @@ func (a *AWS) Serialize(constraints *v1alpha5.Constraints) error { constraints.Provider.Raw = bytes return nil } - -func (a *AWS) GetMetadataOptions() *MetadataOptions { - if a.MetadataOptions == nil { - return &MetadataOptions{ - HTTPEndpoint: aws.String(DefaultMetadataOptionsHTTPEndpoint), - HTTPProtocolIPv6: aws.String(DefaultMetadataOptionsHTTPProtocolIPv6), - HTTPPutResponseHopLimit: aws.Int64(DefaultMetadataOptionsHTTPPutResponseHopLimit), - HTTPTokens: aws.String(DefaultMetadataOptionsHTTPTokens), - } - } - return a.MetadataOptions -} diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go index 93f4c2930968..5e79996195d9 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider_defaults.go @@ -26,7 +26,6 @@ import ( func (c *Constraints) Default(ctx context.Context) { c.defaultArchitecture() c.defaultCapacityTypes() - c.defaultAMIFamily() } func (c *Constraints) defaultCapacityTypes() { @@ -56,13 +55,3 @@ func (c *Constraints) defaultArchitecture() { Values: []string{v1alpha5.ArchitectureAmd64}, }) } - -func (c *Constraints) defaultAMIFamily() { - if c.AMIFamily != nil { - return - } - if c.LaunchTemplate != nil { - return - } - c.AMIFamily = &AMIFamilyAL2 -} diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go b/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go index 317856a6bd14..15fd81f15568 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/provider_validation.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/service/ec2" + "k8s.io/apimachinery/pkg/api/resource" "knative.dev/pkg/apis" ) @@ -29,6 +30,12 @@ const ( amiFamilyPath = "amiFamily" metadataOptionsPath = "metadataOptions" instanceProfilePath = "instanceProfile" + blockDeviceMappingsPath = "blockDeviceMappings" +) + +var ( + minVolumeSize = *resource.NewScaledQuantity(1, resource.Giga) + maxVolumeSize = *resource.NewScaledQuantity(64, resource.Tera) ) func (a *AWS) Validate() (errs *apis.FieldError) { @@ -43,11 +50,12 @@ func (a *AWS) validate() (errs *apis.FieldError) { a.validateTags(), a.validateMetadataOptions(), a.validateAMIFamily(), + a.validateBlockDeviceMappings(), ) } func (a *AWS) validateLaunchTemplate() (errs *apis.FieldError) { - if a.LaunchTemplate == nil { + if a.LaunchTemplateName == nil { return nil } if a.SecurityGroupSelector != nil { @@ -62,6 +70,9 @@ func (a *AWS) validateLaunchTemplate() (errs *apis.FieldError) { if a.InstanceProfile != nil { errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, instanceProfilePath)) } + if len(a.BlockDeviceMappings) != 0 { + errs = errs.Also(apis.ErrMultipleOneOf(launchTemplatePath, blockDeviceMappingsPath)) + } return errs } @@ -78,7 +89,7 @@ func (a *AWS) validateSubnets() (errs *apis.FieldError) { } func (a *AWS) validateSecurityGroups() (errs *apis.FieldError) { - if a.LaunchTemplate != nil { + if a.LaunchTemplateName != nil { return nil } if a.SecurityGroupSelector == nil { @@ -116,14 +127,14 @@ func (a *AWS) validateMetadataOptions() (errs *apis.FieldError) { ).ViaField(metadataOptionsPath) } -func (a *AWS) validateHTTPEndpoint() (errs *apis.FieldError) { +func (a *AWS) validateHTTPEndpoint() *apis.FieldError { if a.MetadataOptions.HTTPEndpoint == nil { return nil } return a.validateStringEnum(*a.MetadataOptions.HTTPEndpoint, "httpEndpoint", ec2.LaunchTemplateInstanceMetadataEndpointState_Values()) } -func (a *AWS) validateHTTPProtocolIpv6() (errs *apis.FieldError) { +func (a *AWS) validateHTTPProtocolIpv6() *apis.FieldError { if a.MetadataOptions.HTTPProtocolIPv6 == nil { return nil } @@ -163,3 +174,54 @@ func (a *AWS) validateStringEnum(value, field string, validValues []string) *api } return apis.ErrInvalidValue(fmt.Sprintf("%s not in %v", value, strings.Join(validValues, ", ")), field) } + +func (a *AWS) validateBlockDeviceMappings() (errs *apis.FieldError) { + for i, blockDeviceMapping := range a.BlockDeviceMappings { + if err := a.validateBlockDeviceMapping(blockDeviceMapping); err != nil { + errs = errs.Also(err.ViaFieldIndex(blockDeviceMappingsPath, i)) + } + } + return errs +} + +func (a *AWS) validateBlockDeviceMapping(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + return errs.Also(a.validateDeviceName(blockDeviceMapping), a.validateEBS(blockDeviceMapping)) +} + +func (a *AWS) validateDeviceName(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.DeviceName == nil { + return apis.ErrMissingField("deviceName") + } + return nil +} + +func (a *AWS) validateEBS(blockDeviceMapping *BlockDeviceMapping) (errs *apis.FieldError) { + if blockDeviceMapping.EBS == nil { + return apis.ErrMissingField("ebs") + } + for _, err := range []*apis.FieldError{ + a.validateVolumeType(blockDeviceMapping), + a.validateVolumeSize(blockDeviceMapping), + } { + if err != nil { + errs = errs.Also(err.ViaField("ebs")) + } + } + return errs +} + +func (a *AWS) validateVolumeType(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.EBS.VolumeType != nil { + return a.validateStringEnum(*blockDeviceMapping.EBS.VolumeType, "volumeType", ec2.VolumeType_Values()) + } + return nil +} + +func (a *AWS) validateVolumeSize(blockDeviceMapping *BlockDeviceMapping) *apis.FieldError { + if blockDeviceMapping.EBS.VolumeSize == nil { + return apis.ErrMissingField("volumeSize") + } else if blockDeviceMapping.EBS.VolumeSize.Cmp(minVolumeSize) == -1 || blockDeviceMapping.EBS.VolumeSize.Cmp(maxVolumeSize) == 1 { + return apis.ErrOutOfBoundsValue(blockDeviceMapping.EBS.VolumeSize.String(), minVolumeSize.String(), maxVolumeSize.String(), "volumeSize") + } + return nil +} diff --git a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go index caa41b1c9853..2084803aa45c 100644 --- a/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/cloudprovider/aws/apis/v1alpha1/zz_generated.deepcopy.go @@ -38,11 +38,6 @@ func (in *AWS) DeepCopyInto(out *AWS) { *out = new(string) **out = **in } - if in.LaunchTemplate != nil { - in, out := &in.LaunchTemplate, &out.LaunchTemplate - *out = new(string) - **out = **in - } if in.SubnetSelector != nil { in, out := &in.SubnetSelector, &out.SubnetSelector *out = make(map[string]string, len(*in)) @@ -64,11 +59,7 @@ func (in *AWS) DeepCopyInto(out *AWS) { (*out)[key] = val } } - if in.MetadataOptions != nil { - in, out := &in.MetadataOptions, &out.MetadataOptions - *out = new(MetadataOptions) - (*in).DeepCopyInto(*out) - } + in.LaunchTemplate.DeepCopyInto(&out.LaunchTemplate) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AWS. @@ -89,6 +80,81 @@ func (in *AWS) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Constraints) DeepCopyInto(out *Constraints) { *out = *in @@ -114,6 +180,42 @@ func (in *Constraints) DeepCopy() *Constraints { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LaunchTemplate) DeepCopyInto(out *LaunchTemplate) { + *out = *in + if in.LaunchTemplateName != nil { + in, out := &in.LaunchTemplateName, &out.LaunchTemplateName + *out = new(string) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LaunchTemplate. +func (in *LaunchTemplate) DeepCopy() *LaunchTemplate { + if in == nil { + return nil + } + out := new(LaunchTemplate) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { *out = *in diff --git a/pkg/cloudprovider/aws/cloudprovider.go b/pkg/cloudprovider/aws/cloudprovider.go index 11f0fcd1c47c..c206b70e8e44 100644 --- a/pkg/cloudprovider/aws/cloudprovider.go +++ b/pkg/cloudprovider/aws/cloudprovider.go @@ -14,6 +14,7 @@ package aws import ( "context" + "encoding/base64" "fmt" "time" @@ -25,26 +26,25 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ssm" + "github.com/patrickmn/go-cache" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/aws/karpenter/pkg/utils/functional" + "github.com/aws/karpenter/pkg/utils/injection" "github.com/aws/karpenter/pkg/utils/project" "go.uber.org/multierr" v1 "k8s.io/api/core/v1" + "k8s.io/client-go/transport" "knative.dev/pkg/apis" "knative.dev/pkg/logging" + "knative.dev/pkg/ptr" ) const ( - // CreationQPS limits the number of requests per second to CreateFleet - // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html#throttling-limits - CreationQPS = 2 - // CreationBurst limits the additional burst requests. - // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html#throttling-limits - CreationBurst = 100 // CacheTTL restricts QPS to AWS APIs to this interval for verifying setup // resources. This value represents the maximum eventual consistency between // AWS actual state and the controller's ability to provision those @@ -89,29 +89,15 @@ func NewCloudProvider(ctx context.Context, options cloudprovider.Options) *Cloud NewLaunchTemplateProvider( ctx, ec2api, - NewAMIProvider(ssm.New(sess), options.ClientSet), + options.ClientSet, + amifamily.New(ssm.New(sess), cache.New(CacheTTL, CacheCleanupInterval)), NewSecurityGroupProvider(ec2api), + getCABundle(ctx), ), }, } } -// get the current region from EC2 IMDS -func getRegionFromIMDS(sess *session.Session) string { - region, err := ec2metadata.New(sess).Region() - if err != nil { - panic(fmt.Sprintf("Failed to call the metadata server's region API, %s", err)) - } - return region -} - -// withUserAgent adds a karpenter specific user-agent string to AWS session -func withUserAgent(sess *session.Session) *session.Session { - userAgent := fmt.Sprintf("karpenter.sh-%s", project.Version) - sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(userAgent)) - return sess -} - // Create a node given the constraints. func (c *CloudProvider) Create(ctx context.Context, constraints *v1alpha5.Constraints, instanceTypes []cloudprovider.InstanceType, quantity int, callback func(*v1.Node) error) error { vendorConstraints, err := v1alpha1.Deserialize(constraints) @@ -170,3 +156,42 @@ func (c *CloudProvider) Default(ctx context.Context, constraints *v1alpha5.Const func (c *CloudProvider) Name() string { return "aws" } + +// get the current region from EC2 IMDS +func getRegionFromIMDS(sess *session.Session) string { + region, err := ec2metadata.New(sess).Region() + if err != nil { + panic(fmt.Sprintf("Failed to call the metadata server's region API, %s", err)) + } + return region +} + +// withUserAgent adds a karpenter specific user-agent string to AWS session +func withUserAgent(sess *session.Session) *session.Session { + userAgent := fmt.Sprintf("karpenter.sh-%s", project.Version) + sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(userAgent)) + return sess +} + +func getCABundle(ctx context.Context) *string { + // Discover CA Bundle from the REST client. We could alternatively + // have used the simpler client-go InClusterConfig() method. + // However, that only works when Karpenter is running as a Pod + // within the same cluster it's managing. + restConfig := injection.GetConfig(ctx) + if restConfig == nil { + return nil + } + transportConfig, err := restConfig.TransportConfig() + if err != nil { + logging.FromContext(ctx).Fatalf("Unable to discover caBundle, loading transport config, %v", err) + return nil + } + _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! + if err != nil { + logging.FromContext(ctx).Fatalf("Unable to discover caBundle, loading TLS config, %v", err) + return nil + } + logging.FromContext(ctx).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) + return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)) +} diff --git a/pkg/cloudprovider/aws/instance.go b/pkg/cloudprovider/aws/instance.go index 7f685134bade..dd6c96350478 100644 --- a/pkg/cloudprovider/aws/instance.go +++ b/pkg/cloudprovider/aws/instance.go @@ -39,6 +39,18 @@ import ( "github.com/aws/karpenter/pkg/utils/options" ) +const ( + // CreationQPS limits the number of requests per second to CreateFleet + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html#throttling-limits + CreationQPS = 2 + // CreationBurst limits the additional burst requests. + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/throttling.html#throttling-limits + CreationBurst = 100 + nvidiaGPUResourceName v1.ResourceName = "nvidia.com/gpu" + amdGPUResourceName v1.ResourceName = "amd.com/gpu" + awsNeuronResourceName v1.ResourceName = "aws.amazon.com/neuron" +) + type InstanceProvider struct { ec2api ec2iface.EC2API instanceTypeProvider *InstanceTypeProvider @@ -46,11 +58,14 @@ type InstanceProvider struct { launchTemplateProvider *LaunchTemplateProvider } -const ( - nvidiaGPUResourceName v1.ResourceName = "nvidia.com/gpu" - amdGPUResourceName v1.ResourceName = "amd.com/gpu" - awsNeuronResourceName v1.ResourceName = "aws.amazon.com/neuron" -) +func NewInstanceProvider(ec2api ec2iface.EC2API, instanceTypeProvider *InstanceTypeProvider, subnetProvider *SubnetProvider, launchTemplateProvider *LaunchTemplateProvider) *InstanceProvider { + return &InstanceProvider{ + ec2api: ec2api, + instanceTypeProvider: instanceTypeProvider, + subnetProvider: subnetProvider, + launchTemplateProvider: launchTemplateProvider, + } +} // Create an instance given the constraints. // instanceTypes should be sorted by priority for spot capacity type. diff --git a/pkg/cloudprovider/aws/launchtemplate.go b/pkg/cloudprovider/aws/launchtemplate.go index 26f239f7e217..79dafb54e13f 100644 --- a/pkg/cloudprovider/aws/launchtemplate.go +++ b/pkg/cloudprovider/aws/launchtemplate.go @@ -15,12 +15,9 @@ limitations under the License. package aws import ( - "bytes" "context" - "encoding/base64" "errors" "fmt" - "sort" "strings" "sync" "time" @@ -31,45 +28,50 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "go.uber.org/zap" - core "k8s.io/api/core/v1" - "k8s.io/client-go/transport" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" "knative.dev/pkg/logging" "knative.dev/pkg/ptr" - "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/aws/karpenter/pkg/utils/functional" "github.com/aws/karpenter/pkg/utils/injection" ) const ( - launchTemplateNameFormat = "Karpenter-%s-%s" + launchTemplateNameFormat = "Karpenter-%s-%s" + kubernetesVersionCacheKey = "kubernetesVersion" ) type LaunchTemplateProvider struct { sync.Mutex - logger *zap.SugaredLogger ec2api ec2iface.EC2API - amiProvider *AMIProvider + clientSet *kubernetes.Clientset + amiFamily *amifamily.Resolver securityGroupProvider *SecurityGroupProvider cache *cache.Cache + logger *zap.SugaredLogger + caBundle *string } -func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, amiProvider *AMIProvider, securityGroupProvider *SecurityGroupProvider) *LaunchTemplateProvider { +func NewLaunchTemplateProvider(ctx context.Context, ec2api ec2iface.EC2API, clientSet *kubernetes.Clientset, amiFamily *amifamily.Resolver, securityGroupProvider *SecurityGroupProvider, caBundle *string) *LaunchTemplateProvider { l := &LaunchTemplateProvider{ ec2api: ec2api, + clientSet: clientSet, logger: logging.FromContext(ctx).Named("launchtemplate"), - amiProvider: amiProvider, + amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, cache: cache.New(CacheTTL, CacheCleanupInterval), + caBundle: caBundle, } l.cache.OnEvicted(l.onCacheEvicted) l.hydrateCache(ctx) return l } -func launchTemplateName(options *launchTemplateOptions) string { +func launchTemplateName(options *amifamily.LaunchTemplate) string { hash, err := hashstructure.Hash(options, hashstructure.FormatV2, nil) if err != nil { panic(fmt.Sprintf("hashing launch template, %s", err)) @@ -77,66 +79,51 @@ func launchTemplateName(options *launchTemplateOptions) string { return fmt.Sprintf(launchTemplateNameFormat, options.ClusterName, fmt.Sprint(hash)) } -// launchTemplateOptions is hashed and results in the creation of a real EC2 -// LaunchTemplate. Do not change this struct without thinking through the impact -// to the number of LaunchTemplates that will result from this change. -type launchTemplateOptions struct { - // Edge-triggered fields that will only change on kube events. - ClusterName string - UserData string - InstanceProfile string - // Level-triggered fields that may change out of sync. - SecurityGroupsIds []string - AMIID string - Tags map[string]string - MetadataOptions *v1alpha1.MetadataOptions -} - func (p *LaunchTemplateProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string) (map[string][]cloudprovider.InstanceType, error) { // If Launch Template is directly specified then just use it - if constraints.LaunchTemplate != nil { - return map[string][]cloudprovider.InstanceType{ptr.StringValue(constraints.LaunchTemplate): instanceTypes}, nil + if constraints.LaunchTemplateName != nil { + return map[string][]cloudprovider.InstanceType{ptr.StringValue(constraints.LaunchTemplateName): instanceTypes}, nil } instanceProfile, err := p.getInstanceProfile(ctx, constraints) if err != nil { return nil, err } // Get constrained security groups - securityGroupsIds, err := p.securityGroupProvider.Get(ctx, constraints) + securityGroupsIDs, err := p.securityGroupProvider.Get(ctx, constraints) if err != nil { return nil, err } - // Get constrained AMI ID - amis, err := p.amiProvider.Get(ctx, constraints, instanceTypes) + kubeServerVersion, err := p.kubeServerVersion(ctx) if err != nil { return nil, err } - // Construct launch templates - launchTemplates := map[string][]cloudprovider.InstanceType{} - caBundle, err := p.GetCABundle(ctx) + resolvedLaunchTemplates, err := p.amiFamily.Resolve(ctx, constraints, instanceTypes, &amifamily.Options{ + ClusterName: injection.GetOptions(ctx).ClusterName, + ClusterEndpoint: injection.GetOptions(ctx).ClusterEndpoint, + AWSENILimitedPodDensity: injection.GetOptions(ctx).AWSENILimitedPodDensity, + InstanceProfile: instanceProfile, + SecurityGroupsIDs: securityGroupsIDs, + Tags: constraints.Tags, + Labels: functional.UnionStringMaps(constraints.Labels, additionalLabels), + CABundle: p.caBundle, + KubernetesVersion: kubeServerVersion, + }) if err != nil { - return nil, fmt.Errorf("getting ca bundle for user data, %w", err) + return nil, err } - for amiID, instanceTypes := range amis { + launchTemplates := map[string][]cloudprovider.InstanceType{} + for _, resolvedLaunchTemplate := range resolvedLaunchTemplates { // Ensure the launch template exists, or create it - launchTemplate, err := p.ensureLaunchTemplate(ctx, constraints, &launchTemplateOptions{ - UserData: p.getUserData(ctx, constraints, instanceTypes, additionalLabels, caBundle), - ClusterName: injection.GetOptions(ctx).ClusterName, - InstanceProfile: instanceProfile, - AMIID: amiID, - SecurityGroupsIds: securityGroupsIds, - Tags: constraints.Tags, - MetadataOptions: constraints.GetMetadataOptions(), - }) + ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, resolvedLaunchTemplate) if err != nil { return nil, err } - launchTemplates[aws.StringValue(launchTemplate.LaunchTemplateName)] = instanceTypes + launchTemplates[*ec2LaunchTemplate.LaunchTemplateName] = resolvedLaunchTemplate.InstanceTypes } return launchTemplates, nil } -func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, constraints *v1alpha1.Constraints, options *launchTemplateOptions) (*ec2.LaunchTemplate, error) { +func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { // Ensure that multiple threads don't attempt to create the same launch template p.Lock() defer p.Unlock() @@ -154,7 +141,7 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, const }) // Create LT if one doesn't exist if isNotFound(err) { - launchTemplate, err = p.createLaunchTemplate(ctx, constraints, options) + launchTemplate, err = p.createLaunchTemplate(ctx, options) if err != nil { return nil, fmt.Errorf("creating launch template, %w", err) } @@ -166,60 +153,20 @@ func (p *LaunchTemplateProvider) ensureLaunchTemplate(ctx context.Context, const logging.FromContext(ctx).Debugf("Discovered launch template %s", name) launchTemplate = output.LaunchTemplates[0] } - // 4. Save in cache to reduce API calls p.cache.SetDefault(name, launchTemplate) return launchTemplate, nil } -// needsDocker returns true if the instance type is unable to use -// containerd directly -func needsDocker(is []cloudprovider.InstanceType) bool { - for _, i := range is { - if !i.AWSNeurons().IsZero() || !i.NvidiaGPUs().IsZero() { - return true - } - } - return false -} - -func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, constraints *v1alpha1.Constraints, options *launchTemplateOptions) (*ec2.LaunchTemplate, error) { - blockDeviceMappings := []*ec2.LaunchTemplateBlockDeviceMappingRequest{{ - DeviceName: aws.String("/dev/xvda"), - Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{ - Encrypted: aws.Bool(true), - VolumeSize: aws.Int64(20), - }, - }} - // Bottlerocket mounts two EBS volumes, /dev/xvda is for the core OS components and Bottlerocket API - // /dev/xvdb is the container image storage, container logs, etc. - if aws.StringValue(constraints.AMIFamily) == v1alpha1.AMIFamilyBottlerocket { - blockDeviceMappings = []*ec2.LaunchTemplateBlockDeviceMappingRequest{ - { - DeviceName: aws.String("/dev/xvda"), - Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{ - Encrypted: aws.Bool(true), - VolumeSize: aws.Int64(4), - }, - }, - { - DeviceName: aws.String("/dev/xvdb"), - Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{ - Encrypted: aws.Bool(true), - VolumeSize: aws.Int64(20), - }, - }, - } - } - +func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { output, err := p.ec2api.CreateLaunchTemplateWithContext(ctx, &ec2.CreateLaunchTemplateInput{ LaunchTemplateName: aws.String(launchTemplateName(options)), LaunchTemplateData: &ec2.RequestLaunchTemplateData{ - BlockDeviceMappings: blockDeviceMappings, + BlockDeviceMappings: p.blockDeviceMappings(options.BlockDeviceMappings), IamInstanceProfile: &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{ Name: aws.String(options.InstanceProfile), }, - SecurityGroupIds: aws.StringSlice(options.SecurityGroupsIds), - UserData: aws.String(options.UserData), + SecurityGroupIds: aws.StringSlice(options.SecurityGroupsIDs), + UserData: aws.String(options.UserData.Script()), ImageId: aws.String(options.AMIID), MetadataOptions: &ec2.LaunchTemplateInstanceMetadataOptionsRequest{ HttpEndpoint: options.MetadataOptions.HTTPEndpoint, @@ -240,6 +187,25 @@ func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, const return output.LaunchTemplate, nil } +func (p *LaunchTemplateProvider) blockDeviceMappings(blockDeviceMappings []*v1alpha1.BlockDeviceMapping) []*ec2.LaunchTemplateBlockDeviceMappingRequest { + blockDeviceMappingsRequest := []*ec2.LaunchTemplateBlockDeviceMappingRequest{} + for _, blockDeviceMapping := range blockDeviceMappings { + blockDeviceMappingsRequest = append(blockDeviceMappingsRequest, &ec2.LaunchTemplateBlockDeviceMappingRequest{ + DeviceName: blockDeviceMapping.DeviceName, + Ebs: &ec2.LaunchTemplateEbsBlockDeviceRequest{ + DeleteOnTermination: blockDeviceMapping.EBS.DeleteOnTermination, + Encrypted: blockDeviceMapping.EBS.Encrypted, + VolumeType: blockDeviceMapping.EBS.VolumeType, + Iops: blockDeviceMapping.EBS.IOPS, + Throughput: blockDeviceMapping.EBS.Throughput, + KmsKeyId: blockDeviceMapping.EBS.KMSKeyID, + VolumeSize: aws.Int64(blockDeviceMapping.EBS.VolumeSize.ScaledValue(resource.Giga)), + }, + }) + } + return blockDeviceMappingsRequest +} + // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *LaunchTemplateProvider) hydrateCache(ctx context.Context) { @@ -259,6 +225,9 @@ func (p *LaunchTemplateProvider) hydrateCache(ctx context.Context) { } func (p *LaunchTemplateProvider) onCacheEvicted(key string, lt interface{}) { + if key == kubernetesVersionCacheKey { + return + } p.Lock() defer p.Unlock() if _, expiration, _ := p.cache.GetWithExpiration(key); expiration.After(time.Now()) { @@ -272,148 +241,6 @@ func (p *LaunchTemplateProvider) onCacheEvicted(key string, lt interface{}) { p.logger.Debugf("Deleted launch template %v", aws.StringValue(launchTemplate.LaunchTemplateId)) } -func sortedTaints(ts []core.Taint) []core.Taint { - sorted := append(ts[:0:0], ts...) // copy to avoid touching original - sort.Slice(sorted, func(i, j int) bool { - ti, tj := sorted[i], sorted[j] - if ti.Key < tj.Key { - return true - } - if ti.Key == tj.Key && ti.Value < tj.Value { - return true - } - if ti.Value == tj.Value { - return ti.Effect < tj.Effect - } - return false - }) - return sorted -} - -func sortedKeys(m map[string]string) []string { - keys := make([]string, len(m)) - i := 0 - for k := range m { - keys[i] = k - i++ - } - sort.Strings(keys) - return keys -} - -func (p *LaunchTemplateProvider) getUserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string, caBundle *string) string { - if aws.StringValue(constraints.AMIFamily) == v1alpha1.AMIFamilyBottlerocket { - return p.getBottlerocketUserData(ctx, constraints, additionalLabels, caBundle) - } - return p.getAL2UserData(ctx, constraints, instanceTypes, additionalLabels, caBundle) -} - -func (p *LaunchTemplateProvider) getBottlerocketUserData(ctx context.Context, constraints *v1alpha1.Constraints, additionalLabels map[string]string, caBundle *string) string { - userData := fmt.Sprintf("[settings.kubernetes]\n\"cluster-name\" = \"%s\"\n\"api-server\" = \"%s\"\n", injection.GetOptions(ctx).ClusterName, injection.GetOptions(ctx).ClusterEndpoint) - if len(constraints.KubeletConfiguration.ClusterDNS) > 0 { - userData += fmt.Sprintf("\"cluster-dns-ip\" = \"%s\"\n", constraints.KubeletConfiguration.ClusterDNS[0]) - } - if caBundle != nil { - userData += fmt.Sprintf("\"cluster-certificate\" = \"%s\"\n", *caBundle) - } - nodeLabelArgs := functional.UnionStringMaps(additionalLabels, constraints.Labels) - if len(nodeLabelArgs) > 0 { - userData += "[settings.kubernetes.node-labels]\n" - for key, val := range nodeLabelArgs { - userData += fmt.Sprintf("\"%s\" = \"%s\"\n", key, val) - } - } - if len(constraints.Taints) > 0 { - userData += "[settings.kubernetes.node-taints]\n" - sorted := sortedTaints(constraints.Taints) - for _, taint := range sorted { - userData += fmt.Sprintf("\"%s\"=\"%s:%s\"\n", taint.Key, taint.Value, taint.Effect) - } - } - return base64.StdEncoding.EncodeToString([]byte(userData)) -} - -// getAL2UserData returns the exact same string for equivalent input, -// even if elements of those inputs are in differing orders, -// guaranteeing it won't cause spurious hash differences. -// AL2 userdata also works on Ubuntu -func (p *LaunchTemplateProvider) getAL2UserData(ctx context.Context, constraints *v1alpha1.Constraints, instanceTypes []cloudprovider.InstanceType, additionalLabels map[string]string, caBundle *string) string { - var containerRuntimeArg string - if !needsDocker(instanceTypes) { - containerRuntimeArg = "--container-runtime containerd" - } - - var userData bytes.Buffer - userData.WriteString(fmt.Sprintf(`#!/bin/bash -xe -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh '%s' %s \ - --apiserver-endpoint '%s'`, - injection.GetOptions(ctx).ClusterName, - containerRuntimeArg, - injection.GetOptions(ctx).ClusterEndpoint)) - if caBundle != nil { - userData.WriteString(fmt.Sprintf(` \ - --b64-cluster-ca '%s'`, - *caBundle)) - } - - nodeLabelArgs := p.getNodeLabelArgs(functional.UnionStringMaps(additionalLabels, constraints.Labels)) - nodeTaintsArgs := p.getNodeTaintArgs(constraints) - kubeletExtraArgs := strings.Trim(strings.Join([]string{nodeLabelArgs, nodeTaintsArgs.String()}, " "), " ") - - if !injection.GetOptions(ctx).AWSENILimitedPodDensity { - userData.WriteString(` \ - --use-max-pods=false`) - kubeletExtraArgs += " --max-pods=110" - } - - if len(kubeletExtraArgs) > 0 { - userData.WriteString(fmt.Sprintf(` \ - --kubelet-extra-args '%s'`, kubeletExtraArgs)) - } - if len(constraints.KubeletConfiguration.ClusterDNS) > 0 { - userData.WriteString(fmt.Sprintf(` \ - --dns-cluster-ip '%s'`, constraints.KubeletConfiguration.ClusterDNS[0])) - } - return base64.StdEncoding.EncodeToString(userData.Bytes()) -} - -func (p *LaunchTemplateProvider) getNodeLabelArgs(nodeLabels map[string]string) string { - nodeLabelArgs := "" - if len(nodeLabels) > 0 { - labelStrings := []string{} - // Must be in sorted order or else equivalent options won't - // hash the same - for _, k := range sortedKeys(nodeLabels) { - if v1alpha5.AllowedLabelDomains.Has(k) { - continue - } - labelStrings = append(labelStrings, fmt.Sprintf("%s=%v", k, nodeLabels[k])) - } - nodeLabelArgs = fmt.Sprintf("--node-labels=%s", strings.Join(labelStrings, ",")) - } - return nodeLabelArgs -} - -func (p *LaunchTemplateProvider) getNodeTaintArgs(constraints *v1alpha1.Constraints) bytes.Buffer { - var nodeTaintsArgs bytes.Buffer - if len(constraints.Taints) > 0 { - nodeTaintsArgs.WriteString("--register-with-taints=") - first := true - // Must be in sorted order or else equivalent options won't - // hash the same. - sorted := sortedTaints(constraints.Taints) - for _, taint := range sorted { - if !first { - nodeTaintsArgs.WriteString(",") - } - first = false - nodeTaintsArgs.WriteString(fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) - } - } - return nodeTaintsArgs -} - func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, constraints *v1alpha1.Constraints) (string, error) { if constraints.InstanceProfile != nil { return aws.StringValue(constraints.InstanceProfile), nil @@ -425,25 +252,16 @@ func (p *LaunchTemplateProvider) getInstanceProfile(ctx context.Context, constra return defaultProfile, nil } -func (p *LaunchTemplateProvider) GetCABundle(ctx context.Context) (*string, error) { - // Discover CA Bundle from the REST client. We could alternatively - // have used the simpler client-go InClusterConfig() method. - // However, that only works when Karpenter is running as a Pod - // within the same cluster it's managing. - restConfig := injection.GetConfig(ctx) - if restConfig == nil { - return nil, nil +func (p *LaunchTemplateProvider) kubeServerVersion(ctx context.Context) (string, error) { + if version, ok := p.cache.Get(kubernetesVersionCacheKey); ok { + return version.(string), nil } - transportConfig, err := restConfig.TransportConfig() + serverVersion, err := p.clientSet.Discovery().ServerVersion() if err != nil { - logging.FromContext(ctx).Debugf("Unable to discover caBundle, loading transport config, %v", err) - return nil, err - } - _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! - if err != nil { - logging.FromContext(ctx).Debugf("Unable to discover caBundle, loading TLS config, %v", err) - return nil, err + return "", err } - logging.FromContext(ctx).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) - return ptr.String(base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData)), nil + version := fmt.Sprintf("%s.%s", serverVersion.Major, strings.TrimSuffix(serverVersion.Minor, "+")) + p.cache.SetDefault(kubernetesVersionCacheKey, version) + logging.FromContext(ctx).Debugf("Discovered kubernetes version %s", version) + return version, nil } diff --git a/pkg/cloudprovider/aws/securitygroups.go b/pkg/cloudprovider/aws/securitygroups.go index ab892d757bf8..b5565ab6ed5c 100644 --- a/pkg/cloudprovider/aws/securitygroups.go +++ b/pkg/cloudprovider/aws/securitygroups.go @@ -40,9 +40,9 @@ func NewSecurityGroupProvider(ec2api ec2iface.EC2API) *SecurityGroupProvider { } } -func (s *SecurityGroupProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints) ([]string, error) { +func (p *SecurityGroupProvider) Get(ctx context.Context, constraints *v1alpha1.Constraints) ([]string, error) { // Get SecurityGroups - securityGroups, err := s.getSecurityGroups(ctx, s.getFilters(constraints)) + securityGroups, err := p.getSecurityGroups(ctx, p.getFilters(constraints)) if err != nil { return nil, err } @@ -58,7 +58,7 @@ func (s *SecurityGroupProvider) Get(ctx context.Context, constraints *v1alpha1.C return securityGroupIds, nil } -func (s *SecurityGroupProvider) getFilters(constraints *v1alpha1.Constraints) []*ec2.Filter { +func (p *SecurityGroupProvider) getFilters(constraints *v1alpha1.Constraints) []*ec2.Filter { filters := []*ec2.Filter{} for key, value := range constraints.SecurityGroupSelector { filters = append(filters, &ec2.Filter{ @@ -69,24 +69,24 @@ func (s *SecurityGroupProvider) getFilters(constraints *v1alpha1.Constraints) [] return filters } -func (s *SecurityGroupProvider) getSecurityGroups(ctx context.Context, filters []*ec2.Filter) ([]*ec2.SecurityGroup, error) { +func (p *SecurityGroupProvider) getSecurityGroups(ctx context.Context, filters []*ec2.Filter) ([]*ec2.SecurityGroup, error) { hash, err := hashstructure.Hash(filters, hashstructure.FormatV2, nil) if err != nil { return nil, err } - if securityGroups, ok := s.cache.Get(fmt.Sprint(hash)); ok { + if securityGroups, ok := p.cache.Get(fmt.Sprint(hash)); ok { return securityGroups.([]*ec2.SecurityGroup), nil } - output, err := s.ec2api.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{Filters: filters}) + output, err := p.ec2api.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{Filters: filters}) if err != nil { return nil, fmt.Errorf("describing security groups %+v, %w", filters, err) } - s.cache.SetDefault(fmt.Sprint(hash), output.SecurityGroups) - logging.FromContext(ctx).Debugf("Discovered security groups: %s", s.securityGroupIds(output.SecurityGroups)) + p.cache.SetDefault(fmt.Sprint(hash), output.SecurityGroups) + logging.FromContext(ctx).Debugf("Discovered security groups: %s", p.securityGroupIds(output.SecurityGroups)) return output.SecurityGroups, nil } -func (s *SecurityGroupProvider) securityGroupIds(securityGroups []*ec2.SecurityGroup) []string { +func (p *SecurityGroupProvider) securityGroupIds(securityGroups []*ec2.SecurityGroup) []string { names := []string{} for _, securityGroup := range securityGroups { names = append(names, aws.StringValue(securityGroup.GroupId)) diff --git a/pkg/cloudprovider/aws/subnets.go b/pkg/cloudprovider/aws/subnets.go index 2a0b2877de8e..bc65d804b0ae 100644 --- a/pkg/cloudprovider/aws/subnets.go +++ b/pkg/cloudprovider/aws/subnets.go @@ -41,23 +41,23 @@ func NewSubnetProvider(ec2api ec2iface.EC2API) *SubnetProvider { } } -func (s *SubnetProvider) Get(ctx context.Context, constraints *v1alpha1.AWS) ([]*ec2.Subnet, error) { +func (p *SubnetProvider) Get(ctx context.Context, constraints *v1alpha1.AWS) ([]*ec2.Subnet, error) { filters := getFilters(constraints) hash, err := hashstructure.Hash(filters, hashstructure.FormatV2, nil) if err != nil { return nil, err } - if subnets, ok := s.cache.Get(fmt.Sprint(hash)); ok { + if subnets, ok := p.cache.Get(fmt.Sprint(hash)); ok { return subnets.([]*ec2.Subnet), nil } - output, err := s.ec2api.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{Filters: filters}) + output, err := p.ec2api.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{Filters: filters}) if err != nil { return nil, fmt.Errorf("describing subnets %s, %w", pretty.Concise(filters), err) } if len(output.Subnets) == 0 { return nil, fmt.Errorf("no subnets matched selector %v", constraints.SubnetSelector) } - s.cache.SetDefault(fmt.Sprint(hash), output.Subnets) + p.cache.SetDefault(fmt.Sprint(hash), output.Subnets) logging.FromContext(ctx).Debugf("Discovered subnets: %s", prettySubnets(output.Subnets)) return output.Subnets, nil } diff --git a/pkg/cloudprovider/aws/suite_test.go b/pkg/cloudprovider/aws/suite_test.go index d728f2ffc373..e8eaef65c165 100644 --- a/pkg/cloudprovider/aws/suite_test.go +++ b/pkg/cloudprovider/aws/suite_test.go @@ -26,6 +26,7 @@ import ( "github.com/Pallinder/go-randomdata" "github.com/aws/amazon-vpc-resource-controller-k8s/pkg/aws/vpc" "github.com/aws/karpenter/pkg/apis/provisioning/v1alpha5" + "github.com/aws/karpenter/pkg/cloudprovider/aws/amifamily" "github.com/aws/karpenter/pkg/cloudprovider/aws/apis/v1alpha1" "github.com/aws/karpenter/pkg/cloudprovider/aws/fake" "github.com/aws/karpenter/pkg/cloudprovider/registry" @@ -49,6 +50,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" . "knative.dev/pkg/logging/testing" + "knative.dev/pkg/ptr" ) var ctx context.Context @@ -57,6 +59,7 @@ var env *test.Environment var launchTemplateCache *cache.Cache var securityGroupCache *cache.Cache var subnetCache *cache.Cache +var amiCache *cache.Cache var unavailableOfferingsCache *cache.Cache var fakeEC2API *fake.EC2API var provisioners *provisioning.Controller @@ -83,6 +86,7 @@ var _ = BeforeSuite(func() { unavailableOfferingsCache = cache.New(InsufficientCapacityErrorCacheTTL, InsufficientCapacityErrorCacheCleanupInterval) securityGroupCache = cache.New(CacheTTL, CacheCleanupInterval) subnetCache = cache.New(CacheTTL, CacheCleanupInterval) + amiCache = cache.New(CacheTTL, CacheCleanupInterval) fakeEC2API = &fake.EC2API{} subnetProvider := &SubnetProvider{ ec2api: fakeEC2API, @@ -94,20 +98,22 @@ var _ = BeforeSuite(func() { cache: cache.New(InstanceTypesAndZonesCacheTTL, CacheCleanupInterval), unavailableOfferings: unavailableOfferingsCache, } - clientSet := kubernetes.NewForConfigOrDie(e.Config) securityGroupProvider := &SecurityGroupProvider{ ec2api: fakeEC2API, cache: securityGroupCache, } + clientSet := kubernetes.NewForConfigOrDie(e.Config) cloudProvider := &CloudProvider{ subnetProvider: subnetProvider, instanceTypeProvider: instanceTypeProvider, instanceProvider: &InstanceProvider{ fakeEC2API, instanceTypeProvider, subnetProvider, &LaunchTemplateProvider{ ec2api: fakeEC2API, - amiProvider: NewAMIProvider(&fake.SSMAPI{}, clientSet), + amiFamily: amifamily.New(fake.SSMAPI{}, amiCache), + clientSet: clientSet, securityGroupProvider: securityGroupProvider, cache: launchTemplateCache, + caBundle: ptr.String("ca-bundle"), }, }, } @@ -133,12 +139,12 @@ var _ = Describe("Allocation", func() { SecurityGroupSelector: map[string]string{"foo": "bar"}, } provisioner = ProvisionerWithProvider(&v1alpha5.Provisioner{ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}}, provider) - provisioner.SetDefaults(ctx) fakeEC2API.Reset() launchTemplateCache.Flush() securityGroupCache.Flush() subnetCache.Flush() unavailableOfferingsCache.Flush() + amiCache.Flush() }) AfterEach(func() { @@ -406,7 +412,7 @@ var _ = Describe("Allocation", func() { }) }) Context("CapacityType", func() { - It("should default to on demand", func() { + It("should default to on-demand", func() { pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod())[0] node := ExpectScheduled(ctx, env.Client, pod) Expect(node.Labels).To(HaveKeyWithValue(v1alpha5.LabelCapacityType, v1alpha1.CapacityTypeOnDemand)) @@ -515,7 +521,8 @@ var _ = Describe("Allocation", func() { Expect(*launchTemplate.Version).To(Equal("$Latest")) }) It("should allow a launch template to be specified", func() { - provider.LaunchTemplate = aws.String("test-launch-template") + provider.LaunchTemplateName = aws.String("test-launch-template") + provider.SecurityGroupSelector = nil pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateFleetInput.Cardinality()).To(Equal(1)) @@ -610,7 +617,7 @@ var _ = Describe("Allocation", func() { Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) userData, _ := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) - Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip '10.0.10.100'")) + Expect(string(userData)).To(ContainSubstring("--dns-cluster-ip='10.0.10.100'")) }) }) Context("Instance Profile", func() { @@ -623,10 +630,10 @@ var _ = Describe("Allocation", func() { Expect(*input.LaunchTemplateData.IamInstanceProfile.Name).To(Equal("test-instance-profile")) }) It("should use the instance profile on the Provisioner when specified", func() { - provider = &v1alpha1.AWS{InstanceProfile: aws.String("overridden-profile")} + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.InstanceProfile = aws.String("overridden-profile") ProvisionerWithProvider(&v1alpha5.Provisioner{ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}}, provider) - provisioner.SetDefaults(ctx) - pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) @@ -641,10 +648,10 @@ var _ = Describe("Allocation", func() { ExpectScheduled(ctx, env.Client, pod) Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) - Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(v1alpha1.DefaultMetadataOptionsHTTPEndpoint)) - Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(v1alpha1.DefaultMetadataOptionsHTTPProtocolIPv6)) - Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(v1alpha1.DefaultMetadataOptionsHTTPPutResponseHopLimit))) - Expect(*input.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(v1alpha1.DefaultMetadataOptionsHTTPTokens)) + Expect(*input.LaunchTemplateData.MetadataOptions.HttpEndpoint).To(Equal(ec2.LaunchTemplateInstanceMetadataEndpointStateEnabled)) + Expect(*input.LaunchTemplateData.MetadataOptions.HttpProtocolIpv6).To(Equal(ec2.LaunchTemplateInstanceMetadataProtocolIpv6Disabled)) + Expect(*input.LaunchTemplateData.MetadataOptions.HttpPutResponseHopLimit).To(Equal(int64(2))) + Expect(*input.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(ec2.LaunchTemplateHttpTokensStateRequired)) }) It("should set metadata options on generated launch template from provisioner configuration", func() { provider, err := ProviderFromProvisioner(provisioner) @@ -665,6 +672,66 @@ var _ = Describe("Allocation", func() { Expect(*input.LaunchTemplateData.MetadataOptions.HttpTokens).To(Equal(ec2.LaunchTemplateHttpTokensStateOptional)) }) }) + Context("Block Device Mappings", func() { + It("should default AL2 block device mappings", func() { + provider, _ := ProviderFromProvisioner(provisioner) + provider.AMIFamily = &v1alpha1.AMIFamilyAL2 + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + ExpectScheduled(ctx, env.Client, pod) + Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) + input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) + Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(20))) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("gp3")) + Expect(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Iops).To(BeNil()) + }) + It("should use custom block device mapping", func() { + provider, _ := ProviderFromProvisioner(provisioner) + provider.AMIFamily = &v1alpha1.AMIFamilyAL2 + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{ + { + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{ + DeleteOnTermination: aws.Bool(true), + Encrypted: aws.Bool(true), + VolumeType: aws.String("io2"), + VolumeSize: resource.NewScaledQuantity(40, resource.Giga), + IOPS: aws.Int64(10_000), + KMSKeyID: aws.String("arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab"), + }, + }, + } + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + ExpectScheduled(ctx, env.Client, pod) + Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) + input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) + Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(40))) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("io2")) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Iops).To(Equal(int64(10_000))) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.DeleteOnTermination).To(BeTrue()) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Encrypted).To(BeTrue()) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.KmsKeyId).To(Equal("arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab")) + }) + It("should default bottlerocket second volume with root volume size", func() { + provider, _ := ProviderFromProvisioner(provisioner) + //provider.BlockDeviceMappings = nil + provider.AMIFamily = &v1alpha1.AMIFamilyBottlerocket + pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, ProvisionerWithProvider(provisioner, provider), test.UnschedulablePod())[0] + ExpectScheduled(ctx, env.Client, pod) + Expect(fakeEC2API.CalledWithCreateLaunchTemplateInput.Cardinality()).To(Equal(1)) + input := fakeEC2API.CalledWithCreateLaunchTemplateInput.Pop().(*ec2.CreateLaunchTemplateInput) + Expect(len(input.LaunchTemplateData.BlockDeviceMappings)).To(Equal(2)) + // Bottlerocket control volume + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(4))) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("gp3")) + Expect(input.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Iops).To(BeNil()) + // Bottlerocket user volume + Expect(*input.LaunchTemplateData.BlockDeviceMappings[1].Ebs.VolumeSize).To(Equal(int64(20))) + Expect(*input.LaunchTemplateData.BlockDeviceMappings[1].Ebs.VolumeType).To(Equal("gp3")) + Expect(input.LaunchTemplateData.BlockDeviceMappings[1].Ebs.Iops).To(BeNil()) + }) + }) }) Context("Defaulting", func() { // Intent here is that if updates occur on the controller, the Provisioner doesn't need to be recreated @@ -705,6 +772,14 @@ var _ = Describe("Allocation", func() { }) }) Context("SecurityGroupSelector", func() { + It("should not allow with a custom launch template", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.LaunchTemplateName = aws.String("my-lt") + provider.SecurityGroupSelector = map[string]string{"key": "value"} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) It("should not allow empty string keys or values", func() { provider, err := ProviderFromProvisioner(provisioner) Expect(err).ToNot(HaveOccurred()) @@ -731,6 +806,14 @@ var _ = Describe("Allocation", func() { }) }) Context("MetadataOptions", func() { + It("should not allow with a custom launch template", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.LaunchTemplateName = aws.String("my-lt") + provider.MetadataOptions = &v1alpha1.MetadataOptions{} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) It("should allow missing values", func() { provider, err := ProviderFromProvisioner(provisioner) Expect(err).ToNot(HaveOccurred()) @@ -835,6 +918,87 @@ var _ = Describe("Allocation", func() { Expect(provisioner.Validate(ctx)).ToNot(Succeed()) }) }) + Context("BlockDeviceMappings", func() { + It("should not allow with a custom launch template", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.LaunchTemplateName = aws.String("my-lt") + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(1, resource.Giga), + }, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + It("should validate minimal device mapping", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(1, resource.Giga), + }, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).To(Succeed()) + }) + It("should not allow volume size below minimum", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(100, resource.Mega), + }, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + It("should not allow volume size above max", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(65, resource.Tera), + }, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + It("should not allow nil device name", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + EBS: &v1alpha1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(65, resource.Tera), + }, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + It("should not allow nil volume size", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + EBS: &v1alpha1.BlockDevice{}, + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + It("should not allow empty ebs block", func() { + provider, err := ProviderFromProvisioner(provisioner) + Expect(err).ToNot(HaveOccurred()) + provider.BlockDeviceMappings = []*v1alpha1.BlockDeviceMapping{{ + DeviceName: aws.String("/dev/xvda"), + }} + provisioner := ProvisionerWithProvider(provisioner, provider) + Expect(provisioner.Validate(ctx)).ToNot(Succeed()) + }) + }) }) }) }) diff --git a/pkg/controllers/metrics/pod/suite_test.go b/pkg/controllers/metrics/pod/suite_test.go index 891b61f0f340..d7552fc00b63 100644 --- a/pkg/controllers/metrics/pod/suite_test.go +++ b/pkg/controllers/metrics/pod/suite_test.go @@ -36,7 +36,7 @@ var env *test.Environment func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "Controllers/Metrics/Node") + RunSpecs(t, "Controllers/Metrics/Pod") } var _ = BeforeSuite(func() { diff --git a/pkg/controllers/provisioning/controller.go b/pkg/controllers/provisioning/controller.go index 1ab0eb5f0b56..eabace105d2e 100644 --- a/pkg/controllers/provisioning/controller.go +++ b/pkg/controllers/provisioning/controller.go @@ -93,6 +93,10 @@ func (c *Controller) Delete(name string) { // Apply creates or updates the provisioner to the latest configuration func (c *Controller) Apply(ctx context.Context, provisioner *v1alpha5.Provisioner) error { + provisioner.SetDefaults(ctx) + if err := provisioner.Validate(ctx); err != nil { + return err + } // Refresh global requirements using instance type availability instanceTypes, err := c.cloudProvider.GetInstanceTypes(ctx, provisioner.Spec.Provider) if err != nil { diff --git a/pkg/controllers/provisioning/scheduling/suite_test.go b/pkg/controllers/provisioning/scheduling/suite_test.go index 615a509cc2ce..73a890ebbf5d 100644 --- a/pkg/controllers/provisioning/scheduling/suite_test.go +++ b/pkg/controllers/provisioning/scheduling/suite_test.go @@ -115,10 +115,10 @@ var _ = Describe("Constraints", func() { }) It("should schedule the pod with Exists operator and defined key", func() { provisioner.Spec.Requirements = v1alpha5.NewRequirements( - v1.NodeSelectorRequirement{Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"bar"}}) + v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}) pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "foo", Operator: v1.NodeSelectorOpExists}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpExists}, }}, ))[0] ExpectScheduled(ctx, env.Client, pod) @@ -133,10 +133,10 @@ var _ = Describe("Constraints", func() { }) It("should not schedule the pod with DoesNotExists operator and defined key", func() { provisioner.Spec.Requirements = v1alpha5.NewRequirements( - v1.NodeSelectorRequirement{Key: "foo", Operator: v1.NodeSelectorOpIn, Values: []string{"bar"}}) + v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1"}}) pod := ExpectProvisioned(ctx, env.Client, selectionController, provisioners, provisioner, test.UnschedulablePod( test.PodOptions{NodeRequirements: []v1.NodeSelectorRequirement{ - {Key: "foo", Operator: v1.NodeSelectorOpDoesNotExist}, + {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpDoesNotExist}, }}, ))[0] ExpectNotScheduled(ctx, env.Client, pod) diff --git a/pkg/controllers/provisioning/suite_test.go b/pkg/controllers/provisioning/suite_test.go index 2b2ff65a72c9..cbf52c0e4073 100644 --- a/pkg/controllers/provisioning/suite_test.go +++ b/pkg/controllers/provisioning/suite_test.go @@ -77,7 +77,6 @@ var _ = Describe("Provisioning", func() { }, }, } - provisioner.SetDefaults(ctx) }) AfterEach(func() { diff --git a/pkg/controllers/selection/suite_test.go b/pkg/controllers/selection/suite_test.go index c59f4c6a7696..2c375bf34e35 100644 --- a/pkg/controllers/selection/suite_test.go +++ b/pkg/controllers/selection/suite_test.go @@ -69,7 +69,6 @@ var _ = BeforeEach(func() { ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, Spec: v1alpha5.ProvisionerSpec{}, } - provisioner.SetDefaults(ctx) }) var _ = AfterEach(func() { diff --git a/website/content/en/preview/AWS/provisioning.md b/website/content/en/preview/AWS/provisioning.md index e3231f7c0c1a..cc279e411bde 100644 --- a/website/content/en/preview/AWS/provisioning.md +++ b/website/content/en/preview/AWS/provisioning.md @@ -165,6 +165,27 @@ spec: amiFamily: Bottlerocket ``` +### Block Device Mappings + +The `blockDeviceMappings` field in a Provisioner can be used to control the Elastic Block Storage (EBS) volumes that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMI Family specified. For example, the `Bottlerocket` AMI Family defaults with two block device mappings, one for Bottlerocket's control volume and the other for container resources such as images and logs. + +Learn more about [block device mappings](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html). + +Note: If a custom launch template is specified, then the `BlockDeviceMappings` field in the launch template is used rather than the provisioner's `blockDeviceMappings`. + +``` +spec: + provider: + blockDeviceMappings: + - deviceName: /dev/xvda + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 +``` ## Other Resources