From 047f8f9aff26683d291d4972e9bda41187be6968 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Tue, 19 Dec 2023 09:35:21 -0800 Subject: [PATCH] docs: Clarify term-based logic in selector sections of EC2NodeClass docs (#5368) --- .../content/en/docs/concepts/nodeclasses.md | 70 +++++++++++- .../en/preview/concepts/nodeclasses.md | 91 ++++++++++++--- .../content/en/v0.32/concepts/nodeclasses.md | 108 ++++++++++++++---- .../content/en/v0.33/concepts/nodeclasses.md | 108 ++++++++++++++---- 4 files changed, 313 insertions(+), 64 deletions(-) diff --git a/website/content/en/docs/concepts/nodeclasses.md b/website/content/en/docs/concepts/nodeclasses.md index 5fd7a7666ff4..5079073cd3d1 100644 --- a/website/content/en/docs/concepts/nodeclasses.md +++ b/website/content/en/docs/concepts/nodeclasses.md @@ -32,14 +32,28 @@ spec: amiFamily: AL2 # Required, discovers subnets to attach to instances + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 # Required, discovers security groups to attach to instances + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c # Optional, IAM role to use for the node identity. # The "role" field is immutable after EC2NodeClass creation. This may change in the @@ -53,9 +67,17 @@ spec: instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" # Optional, discovers amis to override the amiFamily's default + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 # Optional, overrides autogenerated userdata with a merge semantic userData: | @@ -151,6 +173,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.100.0.10' \ --use-max-pods false \ +--container-runtime containerd \ --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` @@ -217,7 +240,19 @@ The `Custom` AMIFamily ships without any default userData to allow you to config ## spec.subnetSelectorTerms -The `EC2NodeClass` discovers subnets through ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` {{% alert title="Tip" color="secondary" %}} Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. @@ -271,9 +306,23 @@ spec: ## spec.securityGroupSelectorTerms -The security group of an instance is comparable to a set of firewall rules. +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. [EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` + {{% alert title="Tip" color="secondary" %}} Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. {{% /alert %}} @@ -345,7 +394,21 @@ spec: ## spec.amiSelectorTerms -AMISelectorTerms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. @@ -650,6 +713,7 @@ Content-Type: text/x-shellscript; charset="us-ascii" exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --use-max-pods false \ +--container-runtime containerd \ --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index d890d84f1d76..3ad9e5ced259 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -32,14 +32,28 @@ spec: amiFamily: AL2 # Required, discovers subnets to attach to instances + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 # Required, discovers security groups to attach to instances + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c # Optional, IAM role to use for the node identity. # The "role" field is immutable after EC2NodeClass creation. This may change in the @@ -53,10 +67,18 @@ spec: instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" # Optional, discovers amis to override the amiFamily's default + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - name: my-ami + - id: ami-123 + # Optional, use instance-store volumes for node ephemeral-storage instanceStorePolicy: RAID0 @@ -220,7 +242,19 @@ The `Custom` AMIFamily ships without any default userData to allow you to config ## spec.subnetSelectorTerms -The `EC2NodeClass` discovers subnets through ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` {{% alert title="Tip" color="secondary" %}} Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. @@ -274,9 +308,23 @@ spec: ## spec.securityGroupSelectorTerms -The security group of an instance is comparable to a set of firewall rules. +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. [EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` + {{% alert title="Tip" color="secondary" %}} Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. {{% /alert %}} @@ -348,7 +396,21 @@ spec: ## spec.amiSelectorTerms -AMISelectorTerms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. @@ -561,28 +623,27 @@ The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.a If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: -```yaml -spec: +```yaml +spec: instanceStorePolicy: RAID0 ``` -This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). -The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: -#### AL2 +#### AL2 -On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. -#### Others +#### Others -For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. -{{% alert title="Tip" color="secondary" %}} -Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. {{% /alert %}} - ## spec.userData You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. diff --git a/website/content/en/v0.32/concepts/nodeclasses.md b/website/content/en/v0.32/concepts/nodeclasses.md index 63f880993988..5079073cd3d1 100644 --- a/website/content/en/v0.32/concepts/nodeclasses.md +++ b/website/content/en/v0.32/concepts/nodeclasses.md @@ -29,21 +29,35 @@ metadata: name: default spec: # Required, resolves a default ami and userdata - amiFamily: AL2 - + amiFamily: AL2 + # Required, discovers subnets to attach to instances - subnetSelectorTerms: + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - id: subnet-09fa4a0a8f233a921 + # Required, discovers security groups to attach to instances - securityGroupSelectorTerms: + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c + # Optional, IAM role to use for the node identity. - # The "role" field is immutable after EC2NodeClass creation. This may change in the - # future, but this restriction is currently in place today to ensure that Karpenter + # The "role" field is immutable after EC2NodeClass creation. This may change in the + # future, but this restriction is currently in place today to ensure that Karpenter # avoids leaking managed instance profiles in your account. # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes role: "KarpenterNodeRole-${CLUSTER_NAME}" @@ -53,19 +67,27 @@ spec: instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" # Optional, discovers amis to override the amiFamily's default - amiSelectorTerms: + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - name: my-ami + - id: ami-123 + # Optional, overrides autogenerated userdata with a merge semantic - userData: | + userData: | echo "Hello world" # Optional, propagates tags to underlying EC2 resources - tags: + tags: team: team-a app: team-a-app - + # Optional, configures IMDS for the instance metadataOptions: httpEndpoint: enabled @@ -103,7 +125,7 @@ status: zone: us-east-2c - id: subnet-03941e7ad6afeaa72 zone: us-east-2a - + # Resolved security groups securityGroups: - id: sg-041513b454818610b @@ -111,7 +133,7 @@ status: - id: sg-0286715698b894bca name: ControlPlaneSecurityGroup-1AQ073TSAAPW - # Resolved AMIs + # Resolved AMIs amis: - id: ami-01234567890123456 name: custom-ami-amd64 @@ -127,7 +149,7 @@ status: operator: In values: - arm64 - + # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` @@ -214,11 +236,23 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https ### Custom -The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. ## spec.subnetSelectorTerms -The `EC2NodeClass` discovers subnets through ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` {{% alert title="Tip" color="secondary" %}} Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. @@ -272,8 +306,22 @@ spec: ## spec.securityGroupSelectorTerms -The security group of an instance is comparable to a set of firewall rules. -[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. +[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` {{% alert title="Tip" color="secondary" %}} Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. @@ -346,9 +394,23 @@ spec: ## spec.amiSelectorTerms -AMISelectorTerms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. -If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. + +If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. {{% alert title="Tip" color="secondary" %}} AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. @@ -900,4 +962,4 @@ spec: role: "KarpenterNodeRole-${CLUSTER_NAME}" status: instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" -``` \ No newline at end of file +``` diff --git a/website/content/en/v0.33/concepts/nodeclasses.md b/website/content/en/v0.33/concepts/nodeclasses.md index 63f880993988..5079073cd3d1 100644 --- a/website/content/en/v0.33/concepts/nodeclasses.md +++ b/website/content/en/v0.33/concepts/nodeclasses.md @@ -29,21 +29,35 @@ metadata: name: default spec: # Required, resolves a default ami and userdata - amiFamily: AL2 - + amiFamily: AL2 + # Required, discovers subnets to attach to instances - subnetSelectorTerms: + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - id: subnet-09fa4a0a8f233a921 + # Required, discovers security groups to attach to instances - securityGroupSelectorTerms: + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c + # Optional, IAM role to use for the node identity. - # The "role" field is immutable after EC2NodeClass creation. This may change in the - # future, but this restriction is currently in place today to ensure that Karpenter + # The "role" field is immutable after EC2NodeClass creation. This may change in the + # future, but this restriction is currently in place today to ensure that Karpenter # avoids leaking managed instance profiles in your account. # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes role: "KarpenterNodeRole-${CLUSTER_NAME}" @@ -53,19 +67,27 @@ spec: instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" # Optional, discovers amis to override the amiFamily's default - amiSelectorTerms: + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" - + environment: test + - name: my-ami + - id: ami-123 + # Optional, overrides autogenerated userdata with a merge semantic - userData: | + userData: | echo "Hello world" # Optional, propagates tags to underlying EC2 resources - tags: + tags: team: team-a app: team-a-app - + # Optional, configures IMDS for the instance metadataOptions: httpEndpoint: enabled @@ -103,7 +125,7 @@ status: zone: us-east-2c - id: subnet-03941e7ad6afeaa72 zone: us-east-2a - + # Resolved security groups securityGroups: - id: sg-041513b454818610b @@ -111,7 +133,7 @@ status: - id: sg-0286715698b894bca name: ControlPlaneSecurityGroup-1AQ073TSAAPW - # Resolved AMIs + # Resolved AMIs amis: - id: ami-01234567890123456 name: custom-ami-amd64 @@ -127,7 +149,7 @@ status: operator: In values: - arm64 - + # Generated instance profile name from "role" instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" ``` @@ -214,11 +236,23 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https ### Custom -The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. ## spec.subnetSelectorTerms -The `EC2NodeClass` discovers subnets through ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` {{% alert title="Tip" color="secondary" %}} Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. @@ -272,8 +306,22 @@ spec: ## spec.securityGroupSelectorTerms -The security group of an instance is comparable to a set of firewall rules. -[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. +[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` {{% alert title="Tip" color="secondary" %}} Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. @@ -346,9 +394,23 @@ spec: ## spec.amiSelectorTerms -AMISelectorTerms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. -If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. + +If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. {{% alert title="Tip" color="secondary" %}} AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. @@ -900,4 +962,4 @@ spec: role: "KarpenterNodeRole-${CLUSTER_NAME}" status: instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" -``` \ No newline at end of file +```