From faec484d66269cfe930bb979e2c686674377d041 Mon Sep 17 00:00:00 2001 From: Ellis Tarn Date: Wed, 30 Jun 2021 09:41:54 -0700 Subject: [PATCH 1/2] Revert "Revert "v1alpha2 API changes (#477)" (#490)" This reverts commit d48734fdff94e272a7672a0b1e0c14b7c9820fd7. --- Makefile | 14 +-- ...rovisioning.karpenter.sh_provisioners.yaml | 14 ++- .../karpenter/templates/webhook/webhooks.yaml | 4 +- cmd/controller/main.go | 6 +- docs/README.md | 104 +++++++++++++----- docs/aws/README.md | 3 +- docs/aws/examples/provisioner.yaml | 12 +- .../designs/metrics-driven-autoscaling.md | 28 ++--- docs/deprecated/designs/scheduled_capacity.md | 4 +- docs/designs/aws-launch-templates-options.md | 12 +- docs/designs/termination.md | 2 +- pkg/apis/apis.go | 6 +- .../{v1alpha1 => v1alpha2}/doc.go | 6 +- .../{v1alpha1 => v1alpha2}/provisioner.go | 29 +++-- .../provisioner_defaults.go | 16 +-- .../provisioner_status.go | 2 +- .../provisioner_validation.go | 12 +- .../provisioner_validation_test.go | 6 +- .../zz_generated.deepcopy.go | 23 ++-- pkg/cloudprovider/aws/cloudprovider.go | 12 +- pkg/cloudprovider/aws/constraints.go | 8 +- pkg/cloudprovider/aws/fake/ec2api.go | 4 +- pkg/cloudprovider/aws/instancetype.go | 4 +- pkg/cloudprovider/aws/launchtemplate.go | 12 +- pkg/cloudprovider/aws/node.go | 4 +- pkg/cloudprovider/aws/securitygroups.go | 4 +- pkg/cloudprovider/aws/subnets.go | 6 +- pkg/cloudprovider/aws/suite_test.go | 24 ++-- pkg/cloudprovider/fake/cloudprovider.go | 6 +- pkg/cloudprovider/registry/register.go | 12 +- pkg/cloudprovider/types.go | 8 +- .../v1alpha1 => }/allocation/bind.go | 4 +- .../v1alpha1 => }/allocation/constraints.go | 8 +- .../v1alpha1 => }/allocation/controller.go | 6 +- .../v1alpha1 => }/allocation/filter.go | 14 +-- .../v1alpha1 => }/allocation/suite_test.go | 32 +++--- pkg/controllers/controller.go | 10 +- .../v1alpha1 => }/reallocation/controller.go | 10 +- .../v1alpha1 => }/reallocation/suite_test.go | 32 +++--- .../v1alpha1 => }/reallocation/utilization.go | 30 ++--- .../v1alpha1 => termination}/controller.go | 4 +- .../v1alpha1 => termination}/suite_test.go | 10 +- .../v1alpha1 => termination}/terminate.go | 4 +- pkg/packing/packer.go | 4 +- pkg/test/expectations/expectations.go | 10 +- pkg/utils/node/predicates.go | 4 +- pkg/utils/ptr/ptr.go | 7 ++ 47 files changed, 338 insertions(+), 258 deletions(-) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/doc.go (91%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/provisioner.go (85%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/provisioner_defaults.go (69%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/provisioner_status.go (98%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/provisioner_validation.go (96%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/provisioner_validation_test.go (98%) rename pkg/apis/provisioning/{v1alpha1 => v1alpha2}/zz_generated.deepcopy.go (92%) rename pkg/controllers/{provisioning/v1alpha1 => }/allocation/bind.go (95%) rename pkg/controllers/{provisioning/v1alpha1 => }/allocation/constraints.go (92%) rename pkg/controllers/{provisioning/v1alpha1 => }/allocation/controller.go (96%) rename pkg/controllers/{provisioning/v1alpha1 => }/allocation/filter.go (90%) rename pkg/controllers/{provisioning/v1alpha1 => }/allocation/suite_test.go (92%) rename pkg/controllers/{provisioning/v1alpha1 => }/reallocation/controller.go (88%) rename pkg/controllers/{provisioning/v1alpha1 => }/reallocation/suite_test.go (77%) rename pkg/controllers/{provisioning/v1alpha1 => }/reallocation/utilization.go (83%) rename pkg/controllers/{terminating/v1alpha1 => termination}/controller.go (98%) rename pkg/controllers/{terminating/v1alpha1 => termination}/suite_test.go (89%) rename pkg/controllers/{terminating/v1alpha1 => termination}/terminate.go (99%) diff --git a/Makefile b/Makefile index 1f0d4db27c2c..7ad86fb8f5d3 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,11 @@ codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/... # CRDs don't currently jive with VolatileTime, which has an Any type. perl -pi -e 's/Any/string/g' charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml hack/boilerplate.sh + gen-crd-api-reference-docs \ + -api-dir ./pkg/apis/provisioning/v1alpha2 \ + -config $(shell go env GOMODCACHE)/github.com/ahmetb/gen-crd-api-reference-docs@v0.2.0/example-config.json \ + -out-file docs/README.md \ + -template-dir $(shell go env GOMODCACHE)/github.com/ahmetb/gen-crd-api-reference-docs@v0.2.0/template publish: ## Generate release manifests and publish a versioned container image. @aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(RELEASE_REPO) @@ -73,14 +78,7 @@ publish: ## Generate release manifests and publish a versioned container image. helm: ## Generate Helm Chart cd charts;helm lint karpenter;helm package karpenter;helm repo index . -docs: ## Generate Docs - gen-crd-api-reference-docs \ - -api-dir ./pkg/apis/provisioning/v1alpha1 \ - -config $(shell go env GOMODCACHE)/github.com/ahmetb/gen-crd-api-reference-docs@v0.2.0/example-config.json \ - -out-file docs/README.md \ - -template-dir $(shell go env GOMODCACHE)/github.com/ahmetb/gen-crd-api-reference-docs@v0.2.0/template - toolchain: ## Install developer toolchain ./hack/toolchain.sh -.PHONY: help dev ci release test battletest verify codegen apply delete publish helm docs toolchain licenses +.PHONY: help dev ci release test battletest verify codegen apply delete publish helm toolchain licenses diff --git a/charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml b/charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml index 81283e158bca..cbab8c638aa2 100644 --- a/charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml +++ b/charts/karpenter/templates/provisioning.karpenter.sh_provisioners.yaml @@ -16,7 +16,7 @@ spec: singular: provisioner scope: Namespaced versions: - - name: v1alpha1 + - name: v1alpha2 schema: openAPIV3Schema: description: Provisioner is the Schema for the Provisioners API @@ -36,7 +36,7 @@ spec: description: Architecture constrains the underlying node architecture type: string cluster: - description: ClusterSpec configures the cluster that the provisioner operates against. If not specified, it will default to using the controller's kube-config. + description: Cluster that launched nodes connect to. properties: caBundle: description: CABundle is required for nodes to verify API Server certificates. @@ -88,9 +88,13 @@ spec: - key type: object type: array - ttlSeconds: - description: TTLSeconds determines how long to wait before attempting to terminate a node. - format: int32 + ttlSecondsAfterEmpty: + description: "TTLSecondsAfterEmpty is the number of seconds the controller will wait before attempting to terminate a node, measured from when the node is detected to be empty. A Node is considered to be empty when it does not have pods scheduled to it, excluding daemonsets. \n Termination due to underutilization is disabled if this field is not set." + format: int64 + type: integer + ttlSecondsUntilExpired: + description: "TTLSecondsUntilExpired is the number of seconds the controller will wait before terminating a node, measured from when the node is created. This is useful to implement features like eventually consistent node upgrade, memory leak protection, and disruption testing. \n Termination due to expiration is disabled if this field is not set." + format: int64 type: integer zones: description: Zones constrains where nodes will be launched by the Provisioner. If unspecified, defaults to all zones in the region. Cannot be specified if label "topology.kubernetes.io/zone" is specified. diff --git a/charts/karpenter/templates/webhook/webhooks.yaml b/charts/karpenter/templates/webhook/webhooks.yaml index 94ee336e5796..84df3981474a 100644 --- a/charts/karpenter/templates/webhook/webhooks.yaml +++ b/charts/karpenter/templates/webhook/webhooks.yaml @@ -15,7 +15,7 @@ webhooks: - apiGroups: - provisioning.karpenter.sh apiVersions: - - v1alpha1 + - v1alpha2 resources: - provisioners provisioners/status @@ -42,7 +42,7 @@ webhooks: - apiGroups: - provisioning.karpenter.sh apiVersions: - - v1alpha1 + - v1alpha2 resources: - provisioners provisioners/status diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 96dd77e8be4a..94e3ad1bc7f2 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -22,9 +22,9 @@ import ( "github.com/awslabs/karpenter/pkg/cloudprovider" "github.com/awslabs/karpenter/pkg/cloudprovider/registry" "github.com/awslabs/karpenter/pkg/controllers" - "github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/allocation" - "github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/reallocation" - termination "github.com/awslabs/karpenter/pkg/controllers/terminating/v1alpha1" + "github.com/awslabs/karpenter/pkg/controllers/allocation" + "github.com/awslabs/karpenter/pkg/controllers/reallocation" + "github.com/awslabs/karpenter/pkg/controllers/termination" "github.com/awslabs/karpenter/pkg/utils/log" "go.uber.org/zap/zapcore" diff --git a/docs/README.md b/docs/README.md index e9b767e0be55..d206353c34c2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,23 +1,23 @@

Packages:

-

provisioning.karpenter.sh/v1alpha1

+

provisioning.karpenter.sh/v1alpha2

-

Package v1alpha1 contains API Schema definitions for the v1alpha1 API group

+

Package v1alpha2 contains API Schema definitions for the v1alpha2 API group

Resource Types: -

ClusterSpec +

Cluster

(Appears on: -ProvisionerSpec) +ProvisionerSpec)

-

ClusterSpec configures the cluster that the provisioner operates against. If +

Cluster configures the cluster that the provisioner operates against. If not specified, it will default to using the controller’s kube-config.

@@ -63,11 +63,11 @@ string
-

Constraints +

Constraints

(Appears on: -ProvisionerSpec) +ProvisionerSpec)

Constraints are applied to all nodes created by the provisioner. They can be @@ -165,7 +165,7 @@ string -

Provisioner +

Provisioner

Provisioner is the Schema for the Provisioners API

@@ -196,7 +196,7 @@ Refer to the Kubernetes API documentation for the fields of the spec
- + ProvisionerSpec @@ -209,20 +209,21 @@ ProvisionerSpec cluster
- -ClusterSpec + +Cluster (Optional) +

Cluster that launched nodes connect to.

Constraints
- + Constraints @@ -231,19 +232,40 @@ Constraints

(Members of Constraints are embedded into this type.)

-

Constraints applied to nodes created by the provisioner

+(Optional) +

Constraints are applied to all nodes launched by this provisioner.

+ + + + +ttlSecondsAfterEmpty
+ +int64 + + + +(Optional) +

TTLSecondsAfterEmpty is the number of seconds the controller will wait +before attempting to terminate a node, measured from when the node is +detected to be empty. A Node is considered to be empty when it does not +have pods scheduled to it, excluding daemonsets.

+

Termination due to underutilization is disabled if this field is not set.

-ttlSeconds
+ttlSecondsUntilExpired
-int32 +int64 (Optional) -

TTLSeconds determines how long to wait before attempting to terminate a node.

+

TTLSecondsUntilExpired is the number of seconds the controller will wait +before terminating a node, measured from when the node is created. This +is useful to implement features like eventually consistent node upgrade, +memory leak protection, and disruption testing.

+

Termination due to expiration is disabled if this field is not set.

@@ -253,7 +275,7 @@ int32 status
- + ProvisionerStatus @@ -263,11 +285,11 @@ ProvisionerStatus -

ProvisionerSpec +

ProvisionerSpec

(Appears on: -Provisioner) +Provisioner)

ProvisionerSpec is the top level provisioner specification. Provisioners @@ -293,20 +315,21 @@ pod.spec.nodeSelector[“provisioning.karpenter.sh/name”]=$PROVISIONER cluster
- -ClusterSpec + +Cluster (Optional) +

Cluster that launched nodes connect to.

Constraints
- + Constraints @@ -315,28 +338,49 @@ Constraints

(Members of Constraints are embedded into this type.)

-

Constraints applied to nodes created by the provisioner

+(Optional) +

Constraints are applied to all nodes launched by this provisioner.

+ + + + +ttlSecondsAfterEmpty
+ +int64 + + + +(Optional) +

TTLSecondsAfterEmpty is the number of seconds the controller will wait +before attempting to terminate a node, measured from when the node is +detected to be empty. A Node is considered to be empty when it does not +have pods scheduled to it, excluding daemonsets.

+

Termination due to underutilization is disabled if this field is not set.

-ttlSeconds
+ttlSecondsUntilExpired
-int32 +int64 (Optional) -

TTLSeconds determines how long to wait before attempting to terminate a node.

+

TTLSecondsUntilExpired is the number of seconds the controller will wait +before terminating a node, measured from when the node is created. This +is useful to implement features like eventually consistent node upgrade, +memory leak protection, and disruption testing.

+

Termination due to expiration is disabled if this field is not set.

-

ProvisionerStatus +

ProvisionerStatus

(Appears on: -Provisioner) +Provisioner)

ProvisionerStatus defines the observed state of Provisioner

@@ -380,5 +424,5 @@ its target, and indicates whether or not those conditions are met.

Generated with gen-crd-api-reference-docs -on git commit 08b78c7. +on git commit 36276c4.

diff --git a/docs/aws/README.md b/docs/aws/README.md index a9be1e501a33..1c0234dca0a8 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -92,7 +92,7 @@ kubectl patch deployment karpenter-controller \ Create a default Provisioner that launches nodes configured with cluster name, endpoint, and caBundle. ```bash cat < Date: Wed, 30 Jun 2021 09:53:30 -0700 Subject: [PATCH 2/2] Verified changes --- docs/README.md | 2 +- docs/aws/README.md | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/README.md b/docs/README.md index d206353c34c2..f382e937cb84 100644 --- a/docs/README.md +++ b/docs/README.md @@ -424,5 +424,5 @@ its target, and indicates whether or not those conditions are met.

Generated with gen-crd-api-reference-docs -on git commit 36276c4. +on git commit faec484.

diff --git a/docs/aws/README.md b/docs/aws/README.md index 1c0234dca0a8..6a9d7dce3bbb 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -97,7 +97,6 @@ kind: Provisioner metadata: name: default spec: - ttlSeconds: 30 cluster: name: ${CLUSTER_NAME} caBundle: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.certificateAuthority.data" --output json)