diff --git a/go.mod b/go.mod index 3dce37d288f9..35a9973acafb 100644 --- a/go.mod +++ b/go.mod @@ -6,18 +6,18 @@ require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/PuerkitoBio/goquery v1.10.0 github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go-v2 v1.32.5 - github.com/aws/aws-sdk-go-v2/config v1.28.5 - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.193.0 - github.com/aws/aws-sdk-go-v2/service/eks v1.52.1 - github.com/aws/aws-sdk-go-v2/service/fis v1.31.1 - github.com/aws/aws-sdk-go-v2/service/iam v1.38.1 - github.com/aws/aws-sdk-go-v2/service/pricing v1.32.6 - github.com/aws/aws-sdk-go-v2/service/sqs v1.37.1 - github.com/aws/aws-sdk-go-v2/service/ssm v1.56.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 - github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.7 + github.com/aws/aws-sdk-go-v2 v1.32.6 + github.com/aws/aws-sdk-go-v2/config v1.28.6 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.195.0 + github.com/aws/aws-sdk-go-v2/service/eks v1.53.0 + github.com/aws/aws-sdk-go-v2/service/fis v1.31.2 + github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 + github.com/aws/aws-sdk-go-v2/service/pricing v1.32.7 + github.com/aws/aws-sdk-go-v2/service/sqs v1.37.2 + github.com/aws/aws-sdk-go-v2/service/ssm v1.56.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 + github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.8 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 github.com/aws/smithy-go v1.22.1 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 @@ -50,15 +50,15 @@ require ( require ( github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.46 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go.sum b/go.sum index 40f25bb884b4..1bbd6415540e 100644 --- a/go.sum +++ b/go.sum @@ -8,48 +8,48 @@ github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsVi github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo= -github.com/aws/aws-sdk-go-v2 v1.32.5/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/config v1.28.5 h1:Za41twdCXbuyyWv9LndXxZZv3QhTG1DinqlFsSuvtI0= -github.com/aws/aws-sdk-go-v2/config v1.28.5/go.mod h1:4VsPbHP8JdcdUDmbTVgNL/8w9SqOkM5jyY8ljIxLO3o= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46 h1:AU7RcriIo2lXjUfHFnFKYsLCwgbz1E7Mm95ieIRDNUg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.46/go.mod h1:1FmYyLGL08KQXQ6mcTlifyFXfJVCNJTVGuQP4m0d/UA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20 h1:sDSXIrlsFSFJtWKLQS4PUWRvrT580rrnuLydJrCQ/yA= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.20/go.mod h1:WZ/c+w0ofps+/OUqMwWgnfrgzZH1DZO1RIkktICsqnY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24 h1:4usbeaes3yJnCFC7kfeyhkdkPtoRYPa/hTmCqMpKpLI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.24/go.mod h1:5CI1JemjVwde8m2WG3cz23qHKPOxbpkq0HaoreEgLIY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24 h1:N1zsICrQglfzaBnrfM0Ys00860C+QFwu6u/5+LomP+o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.24/go.mod h1:dCn9HbJ8+K31i8IQ8EWmWj0EiIk0+vKiHNMxTTYveAg= +github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= +github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/config v1.28.6 h1:D89IKtGrs/I3QXOLNTH93NJYtDhm8SYa9Q5CsPShmyo= +github.com/aws/aws-sdk-go-v2/config v1.28.6/go.mod h1:GDzxJ5wyyFSCoLkS+UhGB0dArhb9mI+Co4dHtoTxbko= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47 h1:48bA+3/fCdi2yAwVt+3COvmatZ6jUDNkDTIsqDiMUdw= +github.com/aws/aws-sdk-go-v2/credentials v1.17.47/go.mod h1:+KdckOejLW3Ks3b0E3b5rHsr2f9yuORBum0WPnE5o5w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 h1:AmoU1pziydclFT/xRV+xXE/Vb8fttJCLRPv8oAkprc0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21/go.mod h1:AjUdLYe4Tgs6kpH4Bv7uMZo7pottoyHMn4eTcIcneaY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 h1:s/fF4+yDQDoElYhfIVvSNyeCydfbuTKzhxSXDXCPasU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25/go.mod h1:IgPfDv5jqFIzQSNbUEMoitNooSMXjRSDkhXv8jiROvU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 h1:ZntTCl5EsYnhN/IygQEUugpdwbhdkom9uHcbCftiGgA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25/go.mod h1:DBdPrgeocww+CSl1C8cEV8PN1mHMBhuCDLpXezyvWkE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.193.0 h1:RhSoBFT5/8tTmIseJUXM6INTXTQDF8+0oyxWBnozIms= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.193.0/go.mod h1:mzj8EEjIHSN2oZRXiw1Dd+uB4HZTl7hC8nBzX9IZMWw= -github.com/aws/aws-sdk-go-v2/service/eks v1.52.1 h1:XqyUdJbXQxY48CbBtN9a51HoTQy/kTIwrWiruRDsydk= -github.com/aws/aws-sdk-go-v2/service/eks v1.52.1/go.mod h1:WTfZ/+I7aSMEna6iYm1Kjne9A8f1MyxXNfp6hCa1+Bk= -github.com/aws/aws-sdk-go-v2/service/fis v1.31.1 h1:bllAkyDhLHCyJFn/nP+b49RRL8Hh/w5Fj4TW8rZhKrw= -github.com/aws/aws-sdk-go-v2/service/fis v1.31.1/go.mod h1:IxUBo1RMVFX9fbptkBFSnqaAh4x5SXdTDmLLsyzYPC4= -github.com/aws/aws-sdk-go-v2/service/iam v1.38.1 h1:hfkzDZHBp9jAT4zcd5mtqckpU4E3Ax0LQaEWWk1VgN8= -github.com/aws/aws-sdk-go-v2/service/iam v1.38.1/go.mod h1:u36ahDtZcQHGmVm/r+0L1sfKX4fzLEMdCqiKRKkUMVM= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.195.0 h1:F3pFi50sK30DZ4IkkNpHwTLGeal5c3nlKuvTgv7xec4= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.195.0/go.mod h1:00zqVNJFK6UASrTnuvjJHJuaqUdkVz5tW8Ip+VhzuNg= +github.com/aws/aws-sdk-go-v2/service/eks v1.53.0 h1:ACTxnLwL6YNmuYbxtp/VR3HGL9SWXU6VZkXPjWST9ZQ= +github.com/aws/aws-sdk-go-v2/service/eks v1.53.0/go.mod h1:ZzOjZXGGUQxOq+T3xmfPLKCZe4OaB5vm1LdGaC8IPn4= +github.com/aws/aws-sdk-go-v2/service/fis v1.31.2 h1:sgNhe7x7r4SffGdtbZteb0AHqCmw5ZHDIiyMCEl6BWs= +github.com/aws/aws-sdk-go-v2/service/fis v1.31.2/go.mod h1:MDGBuQGY9Y4zvv6Bi5tDF4Am+D7fRCvk+nUVndGr0l0= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 h1:8iFKuRj/FJipy/aDZ2lbq0DYuEHdrxp0qVsdi+ZEwnE= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2/go.mod h1:UBe4z0VZnbXGp6xaCW1ulE9pndjfpsnrU206rWZcR0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5 h1:3Y457U2eGukmjYjeHG6kanZpDzJADa2m0ADqnuePYVQ= -github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.5/go.mod h1:CfwEHGkTjYZpkQ/5PvcbEtT7AJlG68KkEvmtwU8z3/U= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5 h1:wtpJ4zcwrSbwhECWQoI/g6WM9zqCcSpHDJIWSbMLOu4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.5/go.mod h1:qu/W9HXQbbQ4+1+JcZp0ZNPV31ym537ZJN+fiS7Ti8E= -github.com/aws/aws-sdk-go-v2/service/pricing v1.32.6 h1:ZzoCQskTXjZBqKW9ZpUFUBCcK22TQZWbO+6PbX8Gu2U= -github.com/aws/aws-sdk-go-v2/service/pricing v1.32.6/go.mod h1:9U+el9JTtl0llHl7GimPXMmqNHkjgMeV9vMVvznTqfs= -github.com/aws/aws-sdk-go-v2/service/sqs v1.37.1 h1:39WvSrVq9DD6UHkD+fx5x19P5KpRQfNdtgReDVNbelc= -github.com/aws/aws-sdk-go-v2/service/sqs v1.37.1/go.mod h1:3gwPzC9LER/BTQdQZ3r6dUktb1rSjABF1D3Sr6nS7VU= -github.com/aws/aws-sdk-go-v2/service/ssm v1.56.0 h1:mADKqoZaodipGgiZfuAjtlcr4IVBtXPZKVjkzUZCCYM= -github.com/aws/aws-sdk-go-v2/service/ssm v1.56.0/go.mod h1:l9qF25TzH95FhcIak6e4vt79KE4I7M2Nf59eMUVjj6c= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6 h1:3zu537oLmsPfDMyjnUS2g+F2vITgy5pB74tHI+JBNoM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.6/go.mod h1:WJSZH2ZvepM6t6jwu4w/Z45Eoi75lPN7DcydSRtJg6Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5 h1:K0OQAsDywb0ltlFrZm0JHPY3yZp/S9OaoLU33S7vPS8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.5/go.mod h1:ORITg+fyuMoeiQFiVGoqB3OydVTLkClw/ljbblMq6Cc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1 h1:6SZUVRQNvExYlMLbHdlKB48x0fLbc2iVROyaNEwBHbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.1/go.mod h1:GqWyYCwLXnlUB1lOAXQyNSPqPLQJvmo8J0DWBzp9mtg= -github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.7 h1:i3Uuh/LLP1Qh3x0qh8i2OnaU+nZ5u3oMwffIrtH7yjc= -github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.7/go.mod h1:UyyUbIDdMO1cGXWA37B/KylPyLrq5kNU1m2E+ojF610= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6 h1:nbmKXZzXPJn41CcD4HsHsGWqvKjLKz9kWu6XxvLmf1s= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.10.6/go.mod h1:SJhcisfKfAawsdNQoZMBEjg+vyN2lH6rO6fP+T94z5Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 h1:50+XsN70RS7dwJ2CkVNXzj7U2L1HKP8nqTd3XWEXBN4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6/go.mod h1:WqgLmwY7so32kG01zD8CPTJWVWM+TzJoOVHwTg4aPug= +github.com/aws/aws-sdk-go-v2/service/pricing v1.32.7 h1:9UDHX1ZgcXUTAGcyxmw04r/6OVG/aUpQ7dZUziR+vTM= +github.com/aws/aws-sdk-go-v2/service/pricing v1.32.7/go.mod h1:68s1DYctoo30LibzEY6gLajXbQEhxpn49+zYFy+Q5Xs= +github.com/aws/aws-sdk-go-v2/service/sqs v1.37.2 h1:mFLfxLZB/TVQwNJAYox4WaxpIu+dFVIcExrmRmRCOhw= +github.com/aws/aws-sdk-go-v2/service/sqs v1.37.2/go.mod h1:GnvfTdlvcpD+or3oslHPOn4Mu6KaCwlCp+0p0oqWnrM= +github.com/aws/aws-sdk-go-v2/service/ssm v1.56.1 h1:cfVjoEwOMOJOI6VoRQua0nI0KjZV9EAnR8bKaMeSppE= +github.com/aws/aws-sdk-go-v2/service/ssm v1.56.1/go.mod h1:fGHwAnTdNrLKhgl+UEeq9uEL4n3Ng4MJucA+7Xi3sC4= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 h1:rLnYAfXQ3YAccocshIH5mzNNwZBkBo+bP6EhIxak6Hw= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.7/go.mod h1:ZHtuQJ6t9A/+YDuxOLnbryAmITtr8UysSny3qcyvJTc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 h1:JnhTZR3PiYDNKlXy50/pNeix9aGMo6lLpXwJ1mw8MD4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6/go.mod h1:URronUEGfXZN1VpdktPSD1EkAL9mfrV+2F4sjH38qOY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HSQqJukaLuuW0TpDA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= +github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.8 h1:chzp64fl/hknlRR9jlstQDB4bYaf848v7KmzUB13omA= +github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.29.8/go.mod h1:6r72p62vXJL+0VTgk9rVV7i9+C0qTcx+HuL56XT9Pus= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 h1:m9rhsGhdepdQV96tZgfy68oU75AWAjOH8u65OefTjwA= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881/go.mod h1:+Mk5k0b6HpKobxNq+B56DOhZ+I/NiPhd5MIBhQMSTSs= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= diff --git a/pkg/providers/instancetype/zz_generated.bandwidth.go b/pkg/providers/instancetype/zz_generated.bandwidth.go index 6cc3e063d570..8861c9ec2a02 100644 --- a/pkg/providers/instancetype/zz_generated.bandwidth.go +++ b/pkg/providers/instancetype/zz_generated.bandwidth.go @@ -35,8 +35,6 @@ var ( // d2.xlarge has vague bandwidth information, bandwidth is Moderate // f1.2xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit // f1.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit - // g3.4xlarge has vague bandwidth information, bandwidth is Up to 10 Gigabit - // g3s.xlarge is not available in https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html // i2.2xlarge has vague bandwidth information, bandwidth is High // i2.4xlarge has vague bandwidth information, bandwidth is High // i2.xlarge has vague bandwidth information, bandwidth is Moderate @@ -165,6 +163,7 @@ var ( "t3.xlarge": 1024, "t3a.xlarge": 1024, "t4g.xlarge": 1024, + "i8g.large": 1172, "a1.xlarge": 1250, "c5.xlarge": 1250, "c5a.xlarge": 1250, @@ -228,12 +227,14 @@ var ( "t3.2xlarge": 2048, "t3a.2xlarge": 2048, "t4g.2xlarge": 2048, + "i7ie.large": 2083, "inf2.xlarge": 2083, "i3en.large": 2100, "m5dn.large": 2100, "m5n.large": 2100, "r5dn.large": 2100, "r5n.large": 2100, + "i8g.xlarge": 2344, "a1.2xlarge": 2500, "c5.2xlarge": 2500, "c5a.2xlarge": 2500, @@ -310,10 +311,12 @@ var ( "m5n.xlarge": 4100, "r5dn.xlarge": 4100, "r5n.xlarge": 4100, + "i7ie.xlarge": 4166, "g4ad.2xlarge": 4167, "i3en.xlarge": 4200, "i4g.2xlarge": 4687, "i4i.2xlarge": 4687, + "i8g.2xlarge": 4688, "a1.4xlarge": 5000, "a1.metal": 5000, "c5.4xlarge": 5000, @@ -400,9 +403,11 @@ var ( "r5dn.2xlarge": 8125, "r5n.2xlarge": 8125, "g4ad.4xlarge": 8333, + "i7ie.2xlarge": 8333, "i3en.2xlarge": 8400, "i4g.4xlarge": 9375, "i4i.4xlarge": 9375, + "i8g.4xlarge": 9375, "c3.8xlarge": 10000, "c4.8xlarge": 10000, "c5a.8xlarge": 10000, @@ -473,6 +478,9 @@ var ( "d3.4xlarge": 12500, "d3en.2xlarge": 12500, "i3en.3xlarge": 12500, + "i7ie.3xlarge": 12500, + "i7ie.6xlarge": 12500, + "i8g.8xlarge": 12500, "im4gn.2xlarge": 12500, "is4gen.2xlarge": 12500, "m6a.8xlarge": 12500, @@ -492,6 +500,7 @@ var ( "r7iz.8xlarge": 12500, "x2iedn.4xlarge": 12500, "x2iezn.2xlarge": 12500, + "i8g.12xlarge": 14063, "c5n.4xlarge": 15000, "c7g.8xlarge": 15000, "c7gd.8xlarge": 15000, @@ -595,6 +604,7 @@ var ( "i3.16xlarge": 25000, "i3.metal": 25000, "i3en.6xlarge": 25000, + "i7ie.12xlarge": 25000, "im4gn.4xlarge": 25000, "inf1.6xlarge": 25000, "is4gen.4xlarge": 25000, @@ -651,6 +661,8 @@ var ( "z1d.12xlarge": 25000, "z1d.metal": 25000, "i4i.12xlarge": 28120, + "i8g.24xlarge": 28125, + "i8g.metal-24xl": 28125, "c7g.16xlarge": 30000, "c7g.metal": 30000, "c7gd.16xlarge": 30000, @@ -676,6 +688,8 @@ var ( "c7i.metal-24xl": 37500, "i4g.16xlarge": 37500, "i4i.16xlarge": 37500, + "i7ie.18xlarge": 37500, + "i8g.16xlarge": 37500, "m6a.24xlarge": 37500, "m6i.24xlarge": 37500, "m6id.24xlarge": 37500, @@ -724,6 +738,7 @@ var ( "g5.24xlarge": 50000, "g6.24xlarge": 50000, "i3en.12xlarge": 50000, + "i7ie.24xlarge": 50000, "im4gn.8xlarge": 50000, "inf2.24xlarge": 50000, "is4gen.8xlarge": 50000, @@ -802,6 +817,7 @@ var ( "hpc6a.48xlarge": 100000, "i3en.24xlarge": 100000, "i3en.metal": 100000, + "i7ie.48xlarge": 100000, "im4gn.16xlarge": 100000, "inf1.24xlarge": 100000, "inf2.48xlarge": 100000, diff --git a/pkg/providers/instancetype/zz_generated.vpclimits.go b/pkg/providers/instancetype/zz_generated.vpclimits.go index 0a43e1c1ab08..1315d01fd33f 100644 --- a/pkg/providers/instancetype/zz_generated.vpclimits.go +++ b/pkg/providers/instancetype/zz_generated.vpclimits.go @@ -17,7 +17,7 @@ // so we can get this information at runtime. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2024-09-20T20:38:24Z +// This file was generated at 2024-11-12T06:00:34Z // WARNING: please add @ellistarn, @bwagner5, or @jonathan-innis from aws/karpenter to reviewers // if you are updating this file since Karpenter is depending on this file to calculate max pods. @@ -2770,6 +2770,186 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "c8g.12xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.16xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.2xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 38, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.48xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.large": { + Interface: 3, + IPv4PerInterface: 10, + IsTrunkingCompatible: true, + BranchInterface: 9, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 3, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.medium": { + Interface: 2, + IPv4PerInterface: 4, + IsTrunkingCompatible: true, + BranchInterface: 4, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 2, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "c8g.metal-24xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "c8g.metal-48xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "c8g.xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 18, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "d2.2xlarge": { Interface: 4, IPv4PerInterface: 15, @@ -7580,6 +7760,186 @@ var Limits = map[string]*VPCLimits{ Hypervisor: "nitro", IsBareMetal: false, }, + "m8g.12xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.16xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.24xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.2xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 38, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.48xlarge": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.4xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.8xlarge": { + Interface: 8, + IPv4PerInterface: 30, + IsTrunkingCompatible: true, + BranchInterface: 54, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 8, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.large": { + Interface: 3, + IPv4PerInterface: 10, + IsTrunkingCompatible: true, + BranchInterface: 9, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 3, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.medium": { + Interface: 2, + IPv4PerInterface: 4, + IsTrunkingCompatible: true, + BranchInterface: 4, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 2, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, + "m8g.metal-24xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "m8g.metal-48xl": { + Interface: 15, + IPv4PerInterface: 50, + IsTrunkingCompatible: true, + BranchInterface: 107, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 15, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "", + IsBareMetal: true, + }, + "m8g.xlarge": { + Interface: 4, + IPv4PerInterface: 15, + IsTrunkingCompatible: true, + BranchInterface: 18, + DefaultNetworkCardIndex: 0, + NetworkCards: []NetworkCard{ + { + MaximumNetworkInterfaces: 4, + NetworkCardIndex: 0, + }, + }, + Hypervisor: "nitro", + IsBareMetal: false, + }, "mac1.metal": { Interface: 8, IPv4PerInterface: 30, diff --git a/pkg/providers/pricing/zz_generated.pricing_aws.go b/pkg/providers/pricing/zz_generated.pricing_aws.go index 9f337299104b..ce7f81e245c6 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2024-11-05T04:33:42Z for us-east-1 +// generated at 2024-12-02T13:14:31Z for us-east-1 import ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -163,6 +163,10 @@ var InitialOnDemandPricesAWS = map[string]map[ec2types.InstanceType]float64{ "i4i.12xlarge": 4.118000, "i4i.16xlarge": 5.491000, "i4i.24xlarge": 8.236800, "i4i.2xlarge": 0.686000, "i4i.32xlarge": 10.982400, "i4i.4xlarge": 1.373000, "i4i.8xlarge": 2.746000, "i4i.large": 0.172000, "i4i.metal": 10.982000, "i4i.xlarge": 0.343000, + // i7ie family + "i7ie.12xlarge": 6.237600, "i7ie.18xlarge": 9.356400, "i7ie.24xlarge": 12.475200, "i7ie.2xlarge": 1.039600, + "i7ie.3xlarge": 1.559400, "i7ie.48xlarge": 24.950400, "i7ie.6xlarge": 3.118800, "i7ie.large": 0.259900, + "i7ie.xlarge": 0.519800, // im4gn family "im4gn.16xlarge": 5.820670, "im4gn.2xlarge": 0.727580, "im4gn.4xlarge": 1.455170, "im4gn.8xlarge": 2.910340, "im4gn.large": 0.181900, "im4gn.xlarge": 0.363790, diff --git a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go index 4f5c25460f42..99b067f5ab0f 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2024-11-05T04:33:46Z for us-east-1 +// generated at 2024-12-02T13:14:37Z for us-east-1 import ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" @@ -250,6 +250,11 @@ var InitialOnDemandPricesUSGov = map[string]map[ec2types.InstanceType]float64{ // g4dn family "g4dn.12xlarge": 4.931000, "g4dn.16xlarge": 5.486000, "g4dn.2xlarge": 0.948000, "g4dn.4xlarge": 1.518000, "g4dn.8xlarge": 2.743000, "g4dn.metal": 9.862000, "g4dn.xlarge": 0.663000, + // g6 family + "g6.12xlarge": 5.800030, "g6.16xlarge": 4.281450, "g6.24xlarge": 8.413670, "g6.2xlarge": 1.232200, + "g6.48xlarge": 16.827340, "g6.4xlarge": 1.667810, "g6.8xlarge": 2.539030, "g6.xlarge": 1.014400, + // gr6 family + "gr6.4xlarge": 1.940100, "gr6.8xlarge": 3.083590, // hpc6a family "hpc6a.48xlarge": 3.467000, // hpc6id family @@ -347,6 +352,8 @@ var InitialOnDemandPricesUSGov = map[string]map[ec2types.InstanceType]float64{ "p3dn.24xlarge": 37.454000, // p4d family "p4d.24xlarge": 39.330000, + // p5 family + "p5.48xlarge": 117.984000, // r3 family "r3.2xlarge": 0.798000, "r3.4xlarge": 1.596000, "r3.8xlarge": 3.192000, "r3.large": 0.200000, "r3.xlarge": 0.399000, diff --git a/website/content/en/docs/concepts/nodepools.md b/website/content/en/docs/concepts/nodepools.md index 15e0da882c5d..f4f900c5a764 100644 --- a/website/content/en/docs/concepts/nodepools.md +++ b/website/content/en/docs/concepts/nodepools.md @@ -254,7 +254,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/docs/troubleshooting.md b/website/content/en/docs/troubleshooting.md index 85e5c1c40599..5c44f8a5f235 100644 --- a/website/content/en/docs/troubleshooting.md +++ b/website/content/en/docs/troubleshooting.md @@ -362,6 +362,84 @@ This can be spot checked like shown above, or monitored via the following metric operator_status_condition_count{type="ConsistentStateFound",kind="NodeClaim",status="False"} ``` +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/docs/upgrading/upgrade-guide.md b/website/content/en/docs/upgrading/upgrade-guide.md index 943d1f66cb82..5d3d33cc2c86 100644 --- a/website/content/en/docs/upgrading/upgrade-guide.md +++ b/website/content/en/docs/upgrading/upgrade-guide.md @@ -17,18 +17,22 @@ With the release of Karpenter v1.0.0, the Karpenter team will be dropping suppor ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - diff --git a/website/content/en/preview/concepts/nodepools.md b/website/content/en/preview/concepts/nodepools.md index 41868765acda..57db6ba21063 100644 --- a/website/content/en/preview/concepts/nodepools.md +++ b/website/content/en/preview/concepts/nodepools.md @@ -72,16 +72,16 @@ spec: # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here - # Note: changing this value in the nodepool will drift the nodeclaims. + # Note: changing this value in the nodepool will drift the nodeclaims. expireAfter: 720h | Never # The amount of time that a node can be draining before it's forcibly deleted. A node begins draining when a delete call is made against it, starting - # its finalization flow. Pods with TerminationGracePeriodSeconds will be deleted preemptively before this terminationGracePeriod ends to give as much time to cleanup as possible. + # its finalization flow. Pods with TerminationGracePeriodSeconds will be deleted preemptively before this terminationGracePeriod ends to give as much time to cleanup as possible. # If your pod's terminationGracePeriodSeconds is larger than this terminationGracePeriod, Karpenter may forcibly delete the pod - # before it has its full terminationGracePeriod to cleanup. + # before it has its full terminationGracePeriod to cleanup. - # Note: changing this value in the nodepool will drift the nodeclaims. - terminationGracePeriod: 48h + # Note: changing this value in the nodepool will drift the nodeclaims. + terminationGracePeriod: 48h # Requirements that constrain the parameters of provisioned nodes. # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. @@ -183,12 +183,12 @@ See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-evic ## spec.template.spec.startupTaints -Taints that are added to nodes to indicate that a certain condition must be met, such as starting an agent or setting up networking, before the node is can be initialized. +Taints that are added to nodes to indicate that a certain condition must be met, such as starting an agent or setting up networking, before the node is can be initialized. These taints must be cleared before pods can be deployed to a node. ## spec.template.spec.expireAfter -The amount of time a Node can live on the cluster before being deleted by Karpenter. Nodes will begin draining once it's expiration has been hit. +The amount of time a Node can live on the cluster before being deleted by Karpenter. Nodes will begin draining once it's expiration has been hit. ## spec.template.spec.terminationGracePeriod @@ -254,7 +254,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/preview/troubleshooting.md b/website/content/en/preview/troubleshooting.md index 85e5c1c40599..5c44f8a5f235 100644 --- a/website/content/en/preview/troubleshooting.md +++ b/website/content/en/preview/troubleshooting.md @@ -362,6 +362,84 @@ This can be spot checked like shown above, or monitored via the following metric operator_status_condition_count{type="ConsistentStateFound",kind="NodeClaim",status="False"} ``` +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/preview/upgrading/upgrade-guide.md b/website/content/en/preview/upgrading/upgrade-guide.md index 943d1f66cb82..5d3d33cc2c86 100644 --- a/website/content/en/preview/upgrading/upgrade-guide.md +++ b/website/content/en/preview/upgrading/upgrade-guide.md @@ -17,18 +17,22 @@ With the release of Karpenter v1.0.0, the Karpenter team will be dropping suppor ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - diff --git a/website/content/en/v0.32/concepts/nodepools.md b/website/content/en/v0.32/concepts/nodepools.md index d5a5a2c424c6..bebf73db87d2 100644 --- a/website/content/en/v0.32/concepts/nodepools.md +++ b/website/content/en/v0.32/concepts/nodepools.md @@ -210,7 +210,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/v0.32/troubleshooting.md b/website/content/en/v0.32/troubleshooting.md index 177043b3c246..be0c954d1eff 100644 --- a/website/content/en/v0.32/troubleshooting.md +++ b/website/content/en/v0.32/troubleshooting.md @@ -414,6 +414,84 @@ However, this should be done with caution. A `VM_MEMORY_OVERHEAD_PERCENT` which results in Karpenter overestimating the memory available on a node can result in Karpenter launching nodes which are too small for your workload. In the worst case, this can result in an instance launch loop and your workload remaining unschedulable indefinitely. +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/v0.32/upgrading/upgrade-guide.md b/website/content/en/v0.32/upgrading/upgrade-guide.md index dc45fc1a1df8..760297aff297 100644 --- a/website/content/en/v0.32/upgrading/upgrade-guide.md +++ b/website/content/en/v0.32/upgrading/upgrade-guide.md @@ -13,29 +13,22 @@ This guide contains information needed to upgrade to the latest release of Karpe ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash - KARPENTER_NAMESPACE=karpenter + KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} -If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. +If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - -In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: - -```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.10/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml -``` - ### Upgrading to v0.32.0+ {{% alert title="Warning" color="warning" %}} diff --git a/website/content/en/v0.37/concepts/nodepools.md b/website/content/en/v0.37/concepts/nodepools.md index 61878c4e8f0a..d93d525cd592 100644 --- a/website/content/en/v0.37/concepts/nodepools.md +++ b/website/content/en/v0.37/concepts/nodepools.md @@ -72,7 +72,7 @@ spec: operator: In values: ["c", "m", "r"] # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods. - # This field is ALPHA and can be dropped or replaced at any time + # This field is ALPHA and can be dropped or replaced at any time minValues: 2 - key: "karpenter.k8s.aws/instance-family" operator: In @@ -229,7 +229,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/v0.37/troubleshooting.md b/website/content/en/v0.37/troubleshooting.md index 11e76125e924..b29670874f64 100644 --- a/website/content/en/v0.37/troubleshooting.md +++ b/website/content/en/v0.37/troubleshooting.md @@ -426,6 +426,84 @@ However, this should be done with caution. A `VM_MEMORY_OVERHEAD_PERCENT` which results in Karpenter overestimating the memory available on a node can result in Karpenter launching nodes which are too small for your workload. In the worst case, this can result in an instance launch loop and your workload remaining unschedulable indefinitely. +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/v0.37/upgrading/upgrade-guide.md b/website/content/en/v0.37/upgrading/upgrade-guide.md index e76a6a204241..a16f204ab3b8 100644 --- a/website/content/en/v0.37/upgrading/upgrade-guide.md +++ b/website/content/en/v0.37/upgrading/upgrade-guide.md @@ -13,26 +13,22 @@ This guide contains information needed to upgrade to the latest release of Karpe ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} -If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. +If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - -In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: - -```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.6/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.6/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.37.6/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml -``` - diff --git a/website/content/en/v1.0/concepts/nodepools.md b/website/content/en/v1.0/concepts/nodepools.md index 2fbc7f959a74..ffde38fc8ddd 100644 --- a/website/content/en/v1.0/concepts/nodepools.md +++ b/website/content/en/v1.0/concepts/nodepools.md @@ -254,7 +254,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/v1.0/troubleshooting.md b/website/content/en/v1.0/troubleshooting.md index 1f0a5c0cfca2..48a324794f5a 100644 --- a/website/content/en/v1.0/troubleshooting.md +++ b/website/content/en/v1.0/troubleshooting.md @@ -353,6 +353,84 @@ This can be spot checked like shown above, or monitored via the following metric operator_status_condition_count{type="ConsistentStateFound",kind="NodeClaim",status="False"} ``` +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/v1.0/upgrading/upgrade-guide.md b/website/content/en/v1.0/upgrading/upgrade-guide.md index ca6ef0bd16d4..544f6f305ea2 100644 --- a/website/content/en/v1.0/upgrading/upgrade-guide.md +++ b/website/content/en/v1.0/upgrading/upgrade-guide.md @@ -17,18 +17,22 @@ With the release of Karpenter v1.0.0, the Karpenter team will be dropping suppor ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - diff --git a/website/content/en/v1.0/upgrading/v1-migration.md b/website/content/en/v1.0/upgrading/v1-migration.md index 1494d934f3bd..2a3d3606581e 100644 --- a/website/content/en/v1.0/upgrading/v1-migration.md +++ b/website/content/en/v1.0/upgrading/v1-migration.md @@ -619,7 +619,7 @@ spec: maxPods: 20 ``` -### NodeClassRef Requirements +#### NodeClassRef Requirements Starting with Karpenter `v1.1.0`, `nodeClassRef.group` and `nodeClassRef.kind` are strictly required on both NodePools and NodeClaims. Ensure these values are set for all resources **before** upgrading Karpenter. diff --git a/website/content/en/v1.1/concepts/nodepools.md b/website/content/en/v1.1/concepts/nodepools.md index 41868765acda..57db6ba21063 100644 --- a/website/content/en/v1.1/concepts/nodepools.md +++ b/website/content/en/v1.1/concepts/nodepools.md @@ -72,16 +72,16 @@ spec: # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here - # Note: changing this value in the nodepool will drift the nodeclaims. + # Note: changing this value in the nodepool will drift the nodeclaims. expireAfter: 720h | Never # The amount of time that a node can be draining before it's forcibly deleted. A node begins draining when a delete call is made against it, starting - # its finalization flow. Pods with TerminationGracePeriodSeconds will be deleted preemptively before this terminationGracePeriod ends to give as much time to cleanup as possible. + # its finalization flow. Pods with TerminationGracePeriodSeconds will be deleted preemptively before this terminationGracePeriod ends to give as much time to cleanup as possible. # If your pod's terminationGracePeriodSeconds is larger than this terminationGracePeriod, Karpenter may forcibly delete the pod - # before it has its full terminationGracePeriod to cleanup. + # before it has its full terminationGracePeriod to cleanup. - # Note: changing this value in the nodepool will drift the nodeclaims. - terminationGracePeriod: 48h + # Note: changing this value in the nodepool will drift the nodeclaims. + terminationGracePeriod: 48h # Requirements that constrain the parameters of provisioned nodes. # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. @@ -183,12 +183,12 @@ See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-evic ## spec.template.spec.startupTaints -Taints that are added to nodes to indicate that a certain condition must be met, such as starting an agent or setting up networking, before the node is can be initialized. +Taints that are added to nodes to indicate that a certain condition must be met, such as starting an agent or setting up networking, before the node is can be initialized. These taints must be cleared before pods can be deployed to a node. ## spec.template.spec.expireAfter -The amount of time a Node can live on the cluster before being deleted by Karpenter. Nodes will begin draining once it's expiration has been hit. +The amount of time a Node can live on the cluster before being deleted by Karpenter. Nodes will begin draining once it's expiration has been hit. ## spec.template.spec.terminationGracePeriod @@ -254,7 +254,9 @@ Karpenter supports `linux` and `windows` operating systems. Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances (note that in this scenario any Spot instances priced higher than the cheapest on-demand instance will be temporarily removed from consideration). +If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 3 minutes. +If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. diff --git a/website/content/en/v1.1/troubleshooting.md b/website/content/en/v1.1/troubleshooting.md index 85e5c1c40599..5c44f8a5f235 100644 --- a/website/content/en/v1.1/troubleshooting.md +++ b/website/content/en/v1.1/troubleshooting.md @@ -362,6 +362,84 @@ This can be spot checked like shown above, or monitored via the following metric operator_status_condition_count{type="ConsistentStateFound",kind="NodeClaim",status="False"} ``` +### Karpenter Is Unable to Satisfy Topology Spread Constraint + +When scheduling pods with TopologySpreadConstraints, Karpenter will attempt to spread the pods across all eligible domains. +Eligible domains are determined based on the pod's requirements, e.g. node affinity terms. +However, pod's do not inherit the requirements of compatible NodePools. + +For example, consider the following NodePool and Deployment specs: + +```yaml +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: Exists +--- +appVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: np-zonal-constraint + labels: + project: zone-specific-project +spec: + template: + spec: + requirements: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] + # ... +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate +spec: + replicas: 3 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + nodeSelector: + project: zone-specific-project + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app: inflate +``` + +This cluster has subnets in three availability zones: `us-east-1a`, `us-east-1b`, and `us-east-1c`. +NodePool `default` can launch instance types in all three zones, but `np-zonal-constraint` is constrained to two. +Since Karpenter uses the pod's requirements to derive eligible domains, and the pod does not have any zonal constraints, all three availability zones are considered eligible domains. +However, the only NodePool compatible with the pod's requirements is `np-zonal-constraints`, which can only create instances in two of the three eligible domains. +Karpenter will succeed to launch the first two instances, for the first two replicas, but will fail to provision capacity for subsequent replicas since it can't provision capacity in the third domain. + +In order to prevent these scenarios, you should ensure that all eligible domains for a pod can be provisioned by compatible NodePools, or constrain the pod such that it's eligble domains match those of the NodePools. +To resolve this specific issue, zonal constraints should be added to the pod spec to match the requirements of `np-zonal-constraint`: +```yaml +nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: ['us-east-1a', 'us-east-1b'] +``` + ## Deprovisioning ### Nodes not deprovisioned diff --git a/website/content/en/v1.1/upgrading/upgrade-guide.md b/website/content/en/v1.1/upgrading/upgrade-guide.md index 943d1f66cb82..5d3d33cc2c86 100644 --- a/website/content/en/v1.1/upgrading/upgrade-guide.md +++ b/website/content/en/v1.1/upgrading/upgrade-guide.md @@ -17,18 +17,22 @@ With the release of Karpenter v1.0.0, the Karpenter team will be dropping suppor ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd)) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) ([source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds)). + Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/) - the tool will only install the CRD during the first installation of the Helm chart. + Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. + +CRDs are coupled to the version of Karpenter, and should be updated along with Karpenter. +For this reason, we recommend using the independent `karpenter-crd` chart to manage CRDs. {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. -