Revert "Revert "v1alpha2 API changes (#477)" (#490)"

Makefile

@@ -63,6 +63,11 @@ codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/...
 	# CRDs don't currently jive with VolatileTime, which has an Any type.
 	perl -pi -e 's/Any/string/g' charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml
 	hack/boilerplate.sh
+	gen-crd-api-reference-docs \
+		-api-dir ./pkg/apis/provisioning/v1alpha2 \
+		-config $(shell go env GOMODCACHE)/github.com/ahmetb/[email protected]/example-config.json \
+		-out-file docs/README.md \
+		-template-dir $(shell go env GOMODCACHE)/github.com/ahmetb/[email protected]/template
 
 publish: ## Generate release manifests and publish a versioned container image.
 	@aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(RELEASE_REPO)
@@ -73,14 +78,7 @@ publish: ## Generate release manifests and publish a versioned container image.
 helm:  ## Generate Helm Chart
 	cd charts;helm lint karpenter;helm package karpenter;helm repo index .
 
-docs: ## Generate Docs
-	gen-crd-api-reference-docs \
-		-api-dir ./pkg/apis/provisioning/v1alpha1 \
-		-config $(shell go env GOMODCACHE)/github.com/ahmetb/[email protected]/example-config.json \
-		-out-file docs/README.md \
-		-template-dir $(shell go env GOMODCACHE)/github.com/ahmetb/[email protected]/template
-
 toolchain: ## Install developer toolchain
 	./hack/toolchain.sh
 
-.PHONY: help dev ci release test battletest verify codegen apply delete publish helm docs toolchain licenses
+.PHONY: help dev ci release test battletest verify codegen apply delete publish helm toolchain licenses

charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml

@@ -16,7 +16,7 @@ spec:
     singular: provisioner
   scope: Namespaced
   versions:
-  - name: v1alpha1
+  - name: v1alpha2
     schema:
       openAPIV3Schema:
         description: Provisioner is the Schema for the Provisioners API
@@ -36,7 +36,7 @@ spec:
                 description: Architecture constrains the underlying node architecture
                 type: string
               cluster:
-                description: ClusterSpec configures the cluster that the provisioner operates against. If not specified, it will default to using the controller's kube-config.
+                description: Cluster that launched nodes connect to.
                 properties:
                   caBundle:
                     description: CABundle is required for nodes to verify API Server certificates.
@@ -88,9 +88,13 @@ spec:
                   - key
                   type: object
                 type: array
-              ttlSeconds:
-                description: TTLSeconds determines how long to wait before attempting to terminate a node.
-                format: int32
+              ttlSecondsAfterEmpty:
+                description: "TTLSecondsAfterEmpty is the number of seconds the controller will wait before attempting to terminate a node, measured from when the node is detected to be empty. A Node is considered to be empty when it does not have pods scheduled to it, excluding daemonsets. \n Termination due to underutilization is disabled if this field is not set."
+                format: int64
+                type: integer
+              ttlSecondsUntilExpired:
+                description: "TTLSecondsUntilExpired is the number of seconds the controller will wait before terminating a node, measured from when the node is created. This is useful to implement features like eventually consistent node upgrade, memory leak protection, and disruption testing. \n Termination due to expiration is disabled if this field is not set."
+                format: int64
                 type: integer
               zones:
                 description: Zones constrains where nodes will be launched by the Provisioner. If unspecified, defaults to all zones in the region. Cannot be specified if label "topology.kubernetes.io/zone" is specified.

charts/karpenter/templates/webhook/webhooks.yaml

@@ -15,7 +15,7 @@ webhooks:
   - apiGroups:
     - provisioning.karpenter.sh
     apiVersions:
-    - v1alpha1
+    - v1alpha2
     resources:
     - provisioners
       provisioners/status
@@ -42,7 +42,7 @@ webhooks:
   - apiGroups:
     - provisioning.karpenter.sh
     apiVersions:
-    - v1alpha1
+    - v1alpha2
     resources:
     - provisioners
       provisioners/status

cmd/controller/main.go

@@ -22,9 +22,9 @@ import (
 	"github.com/awslabs/karpenter/pkg/cloudprovider"
 	"github.com/awslabs/karpenter/pkg/cloudprovider/registry"
 	"github.com/awslabs/karpenter/pkg/controllers"
-	"github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/allocation"
-	"github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/reallocation"
-	termination "github.com/awslabs/karpenter/pkg/controllers/terminating/v1alpha1"
+	"github.com/awslabs/karpenter/pkg/controllers/allocation"
+	"github.com/awslabs/karpenter/pkg/controllers/reallocation"
+	"github.com/awslabs/karpenter/pkg/controllers/termination"
 	"github.com/awslabs/karpenter/pkg/utils/log"
 
 	"go.uber.org/zap/zapcore"

docs/README.md

@@ -1,23 +1,23 @@
 <p>Packages:</p>
 <ul>
 <li>
-<a href="#provisioning.karpenter.sh%2fv1alpha1">provisioning.karpenter.sh/v1alpha1</a>
+<a href="#provisioning.karpenter.sh%2fv1alpha2">provisioning.karpenter.sh/v1alpha2</a>
 </li>
 </ul>
-<h2 id="provisioning.karpenter.sh/v1alpha1">provisioning.karpenter.sh/v1alpha1</h2>
+<h2 id="provisioning.karpenter.sh/v1alpha2">provisioning.karpenter.sh/v1alpha2</h2>
 <p>
-<p>Package v1alpha1 contains API Schema definitions for the v1alpha1 API group</p>
+<p>Package v1alpha2 contains API Schema definitions for the v1alpha2 API group</p>
 </p>
 Resource Types:
 <ul></ul>
-<h3 id="provisioning.karpenter.sh/v1alpha1.ClusterSpec">ClusterSpec
+<h3 id="provisioning.karpenter.sh/v1alpha2.Cluster">Cluster
 </h3>
 <p>
 (<em>Appears on:</em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ProvisionerSpec">ProvisionerSpec</a>)
+<a href="#provisioning.karpenter.sh/v1alpha2.ProvisionerSpec">ProvisionerSpec</a>)
 </p>
 <p>
-<p>ClusterSpec configures the cluster that the provisioner operates against. If
+<p>Cluster configures the cluster that the provisioner operates against. If
 not specified, it will default to using the controller&rsquo;s kube-config.</p>
 </p>
 <table>
@@ -63,11 +63,11 @@ string
 </tr>
 </tbody>
 </table>
-<h3 id="provisioning.karpenter.sh/v1alpha1.Constraints">Constraints
+<h3 id="provisioning.karpenter.sh/v1alpha2.Constraints">Constraints
 </h3>
 <p>
 (<em>Appears on:</em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ProvisionerSpec">ProvisionerSpec</a>)
+<a href="#provisioning.karpenter.sh/v1alpha2.ProvisionerSpec">ProvisionerSpec</a>)
 </p>
 <p>
 <p>Constraints are applied to all nodes created by the provisioner. They can be
@@ -165,7 +165,7 @@ string
 </tr>
 </tbody>
 </table>
-<h3 id="provisioning.karpenter.sh/v1alpha1.Provisioner">Provisioner
+<h3 id="provisioning.karpenter.sh/v1alpha2.Provisioner">Provisioner
 </h3>
 <p>
 <p>Provisioner is the Schema for the Provisioners API</p>
@@ -196,7 +196,7 @@ Refer to the Kubernetes API documentation for the fields of the
 <td>
 <code>spec</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ProvisionerSpec">
+<a href="#provisioning.karpenter.sh/v1alpha2.ProvisionerSpec">
 ProvisionerSpec
 </a>
 </em>
@@ -209,20 +209,21 @@ ProvisionerSpec
 <td>
 <code>cluster</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ClusterSpec">
-ClusterSpec
+<a href="#provisioning.karpenter.sh/v1alpha2.Cluster">
+Cluster
 </a>
 </em>
 </td>
 <td>
 <em>(Optional)</em>
+<p>Cluster that launched nodes connect to.</p>
 </td>
 </tr>
 <tr>
 <td>
 <code>Constraints</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.Constraints">
+<a href="#provisioning.karpenter.sh/v1alpha2.Constraints">
 Constraints
 </a>
 </em>
@@ -231,19 +232,40 @@ Constraints
 <p>
 (Members of <code>Constraints</code> are embedded into this type.)
 </p>
-<p>Constraints applied to nodes created by the provisioner</p>
+<em>(Optional)</em>
+<p>Constraints are applied to all nodes launched by this provisioner.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>ttlSecondsAfterEmpty</code></br>
+<em>
+int64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>TTLSecondsAfterEmpty is the number of seconds the controller will wait
+before attempting to terminate a node, measured from when the node is
+detected to be empty. A Node is considered to be empty when it does not
+have pods scheduled to it, excluding daemonsets.</p>
+<p>Termination due to underutilization is disabled if this field is not set.</p>
 </td>
 </tr>
 <tr>
 <td>
-<code>ttlSeconds</code></br>
+<code>ttlSecondsUntilExpired</code></br>
 <em>
-int32
+int64
 </em>
 </td>
 <td>
 <em>(Optional)</em>
-<p>TTLSeconds determines how long to wait before attempting to terminate a node.</p>
+<p>TTLSecondsUntilExpired is the number of seconds the controller will wait
+before terminating a node, measured from when the node is created. This
+is useful to implement features like eventually consistent node upgrade,
+memory leak protection, and disruption testing.</p>
+<p>Termination due to expiration is disabled if this field is not set.</p>
 </td>
 </tr>
 </table>
@@ -253,7 +275,7 @@ int32
 <td>
 <code>status</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ProvisionerStatus">
+<a href="#provisioning.karpenter.sh/v1alpha2.ProvisionerStatus">
 ProvisionerStatus
 </a>
 </em>
@@ -263,11 +285,11 @@ ProvisionerStatus
 </tr>
 </tbody>
 </table>
-<h3 id="provisioning.karpenter.sh/v1alpha1.ProvisionerSpec">ProvisionerSpec
+<h3 id="provisioning.karpenter.sh/v1alpha2.ProvisionerSpec">ProvisionerSpec
 </h3>
 <p>
 (<em>Appears on:</em>
-<a href="#provisioning.karpenter.sh/v1alpha1.Provisioner">Provisioner</a>)
+<a href="#provisioning.karpenter.sh/v1alpha2.Provisioner">Provisioner</a>)
 </p>
 <p>
 <p>ProvisionerSpec is the top level provisioner specification. Provisioners
@@ -293,20 +315,21 @@ pod.spec.nodeSelector[&ldquo;provisioning.karpenter.sh/name&rdquo;]=$PROVISIONER
 <td>
 <code>cluster</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.ClusterSpec">
-ClusterSpec
+<a href="#provisioning.karpenter.sh/v1alpha2.Cluster">
+Cluster
 </a>
 </em>
 </td>
 <td>
 <em>(Optional)</em>
+<p>Cluster that launched nodes connect to.</p>
 </td>
 </tr>
 <tr>
 <td>
 <code>Constraints</code></br>
 <em>
-<a href="#provisioning.karpenter.sh/v1alpha1.Constraints">
+<a href="#provisioning.karpenter.sh/v1alpha2.Constraints">
 Constraints
 </a>
 </em>
@@ -315,28 +338,49 @@ Constraints
 <p>
 (Members of <code>Constraints</code> are embedded into this type.)
 </p>
-<p>Constraints applied to nodes created by the provisioner</p>
+<em>(Optional)</em>
+<p>Constraints are applied to all nodes launched by this provisioner.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>ttlSecondsAfterEmpty</code></br>
+<em>
+int64
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>TTLSecondsAfterEmpty is the number of seconds the controller will wait
+before attempting to terminate a node, measured from when the node is
+detected to be empty. A Node is considered to be empty when it does not
+have pods scheduled to it, excluding daemonsets.</p>
+<p>Termination due to underutilization is disabled if this field is not set.</p>
 </td>
 </tr>
 <tr>
 <td>
-<code>ttlSeconds</code></br>
+<code>ttlSecondsUntilExpired</code></br>
 <em>
-int32
+int64
 </em>
 </td>
 <td>
 <em>(Optional)</em>
-<p>TTLSeconds determines how long to wait before attempting to terminate a node.</p>
+<p>TTLSecondsUntilExpired is the number of seconds the controller will wait
+before terminating a node, measured from when the node is created. This
+is useful to implement features like eventually consistent node upgrade,
+memory leak protection, and disruption testing.</p>
+<p>Termination due to expiration is disabled if this field is not set.</p>
 </td>
 </tr>
 </tbody>
 </table>
-<h3 id="provisioning.karpenter.sh/v1alpha1.ProvisionerStatus">ProvisionerStatus
+<h3 id="provisioning.karpenter.sh/v1alpha2.ProvisionerStatus">ProvisionerStatus
 </h3>
 <p>
 (<em>Appears on:</em>
-<a href="#provisioning.karpenter.sh/v1alpha1.Provisioner">Provisioner</a>)
+<a href="#provisioning.karpenter.sh/v1alpha2.Provisioner">Provisioner</a>)
 </p>
 <p>
 <p>ProvisionerStatus defines the observed state of Provisioner</p>
@@ -380,5 +424,5 @@ its target, and indicates whether or not those conditions are met.</p>
 <hr/>
 <p><em>
 Generated with <code>gen-crd-api-reference-docs</code>
-on git commit <code>08b78c7</code>.
+on git commit <code>faec484</code>.
 </em></p>

docs/aws/README.md

@@ -92,16 +92,16 @@ kubectl patch deployment karpenter-controller \
 Create a default Provisioner that launches nodes configured with cluster name, endpoint, and caBundle.
 ```bash
 cat <<EOF | kubectl apply -f -
-apiVersion: provisioning.karpenter.sh/v1alpha1
+apiVersion: provisioning.karpenter.sh/v1alpha2
 kind: Provisioner
 metadata:
   name: default
 spec:
-  ttlSeconds: 30
   cluster:
     name: ${CLUSTER_NAME}
     caBundle: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.certificateAuthority.data" --output json)
     endpoint: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json)
+  ttlSecondsAfterEmpty: 30
 EOF
 kubectl get provisioner default -o yaml
 ```

docs/aws/examples/provisioner.yaml

@@ -3,14 +3,24 @@ kind: Provisioner
 metadata:
   name: default
 spec:
-  # Provisioned nodes will be configured to connect to this cluster
+  # Provisioned nodes connect to this cluster
   cluster:
     name: "${CLUSTER_NAME}"
     caBundle: "${CLUSTER_CA_BUNDLE}"
     endpoint: "${CLUSTER_ENDPOINT}"
+
+  # If nil, the feature is disabled, nodes will never expire
+  ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds;
+
+  # If nil, the feature is disabled, nodes will never scale down due to low utilization
+  ttlSecondsAfterEmpty: 30
+
+  # Provisioned nodes will have these taints
   taints:
     - key: example.com/special-taint
       effect: NoSchedule
+
+  # Provisioned nodes will have these labels
   labels:
     ##### AWS Specific #####
     # Constrain node launch template, default="bottlerocket"