diff --git a/.github/workflows/presubmit.yaml b/.github/workflows/presubmit.yaml index 631b2a534d17..15917e01d56f 100644 --- a/.github/workflows/presubmit.yaml +++ b/.github/workflows/presubmit.yaml @@ -1,7 +1,9 @@ name: Presubmit on: pull_request: - branches: [ master ] + branches: + - main + - provisioner-work jobs: build: runs-on: ubuntu-latest @@ -11,7 +13,7 @@ jobs: with: go-version: 1.15.3 - run: make toolchain - - run: echo ::add-path::/usr/local/kubebuilder/bin + - run: echo "/usr/local/kubebuilder/bin" >> $GITHUB_PATH - run: make ci - uses: actions/upload-artifact@v2 with: diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1789040cae97..000000000000 --- a/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM golang:1.15.3 as builder - -# Copy src -WORKDIR /go/src/github.com/awslabs/karpenter -COPY go.mod go.mod -COPY go.sum go.sum - -# Build src -RUN GOPROXY=direct go mod download - -COPY karpenter/ karpenter/ -COPY pkg/ pkg/ - -RUN go build -o karpenter ./karpenter - -# Copy to slim image -FROM gcr.io/distroless/base:latest -WORKDIR / -COPY --from=builder /go/src/github.com/awslabs/karpenter . -ENTRYPOINT ["/karpenter"] diff --git a/Makefile b/Makefile index 6487e1621f3c..2df0cfc113b0 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,27 @@ GOFLAGS ?= "-tags=${CLOUD_PROVIDER}" + +RELEASE_REPO ?= public.ecr.aws/b6u6q9h4 +RELEASE_VERSION ?= v0.1.2 +RELEASE_MANIFEST = releases/${CLOUD_PROVIDER}/manifest.yaml + WITH_GOFLAGS = GOFLAGS=${GOFLAGS} -RELEASE_VERSION ?= v0.1.0 +WITH_RELEASE_REPO = KO_DOCKER_REPO=${RELEASE_REPO} help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) -all: generate verify test ## Run all steps in the developer loop +dev: codegen verify test ## Run all steps in the developer loop -ci: generate verify battletest ## Run all steps used by continuous integration +ci: codegen verify battletest ## Run all steps used by continuous integration + +release: publish helm docs ## Run all steps in release workflow test: ## Run tests ginkgo -r battletest: ## Run stronger tests # Ensure all files have cyclo-complexity =< 10 - gocyclo -over 10 ./pkg + gocyclo -over 11 ./pkg # Run randomized, parallelized, racing, code coveraged, tests ginkgo -r \ -cover -coverprofile=coverage.out -outputdir=. -coverpkg=./pkg/... \ @@ -28,7 +35,7 @@ verify: ## Verify code. Includes dependencies, linting, formatting, etc go fmt ./... golangci-lint run -generate: ## Generate code. Must be run if changes are made to ./pkg/apis/... +codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/... controller-gen \ object:headerFile="hack/boilerplate.go.txt" \ webhook \ @@ -47,6 +54,7 @@ generate: ## Generate code. Must be run if changes are made to ./pkg/apis/... perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_horizontalautoscalers.yaml perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_scalablenodegroups.yaml perl -pi -e 's/Any/string/g' config/crd/bases/autoscaling.karpenter.sh_metricsproducers.yaml + perl -pi -e 's/Any/string/g' config/crd/bases/provisioning.karpenter.sh_provisioners.yaml apply: ## Deploy the controller into your ~/.kube/config cluster kubectl kustomize config | $(WITH_GOFLAGS) ko apply -B -f - @@ -54,8 +62,13 @@ apply: ## Deploy the controller into your ~/.kube/config cluster delete: ## Delete the controller from your ~/.kube/config cluster kubectl kustomize config | ko delete -f - -release: ## Publish a versioned container image to $KO_DOCKER_REPO/karpenter and generate release manifests. - kubectl kustomize config | $(WITH_GOFLAGS) ko resolve -B -t $(RELEASE_VERSION) -f - > releases/${CLOUD_PROVIDER}/$(RELEASE_VERSION).yaml +publish: ## Generate release manifests and publish a versioned container image. + kubectl kustomize config | $(WITH_RELEASE_REPO) $(WITH_GOFLAGS) ko resolve -B -t $(RELEASE_VERSION) -f - > $(RELEASE_MANIFEST) + +helm: ## Generate Helm Chart + cp $(RELEASE_MANIFEST) charts/karpenter/templates + yq w -i charts/karpenter/Chart.yaml version $(RELEASE_VERSION) + cd charts; helm package karpenter; helm repo index . docs: ## Generate Docs gen-crd-api-reference-docs \ @@ -67,4 +80,4 @@ docs: ## Generate Docs toolchain: ## Install developer toolchain ./hack/toolchain.sh -.PHONY: help all ci test release run apply delete verify generate docs toolchain +.PHONY: help dev ci release test battletest verify codegen apply delete publish helm docs toolchain diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000000..63c76cafe726 --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +Karpenter +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/OWNERS b/OWNERS index 2fc549116006..639dbe6f5aa9 100644 --- a/OWNERS +++ b/OWNERS @@ -1,3 +1,7 @@ approvers: - ellistarn - JacobGabrielson + - tabern +reviewers: + - prateekgogia + - njtran diff --git a/README.md b/README.md index 316721a4f331..929f591e6205 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,20 @@ -# Karpenter -![](./docs/images/logo.jpeg) +![](./docs/images/karpenter-banner.png) -Karpenter is a metrics-driven autoscaler for Kubernetes. It's performant, extensible, and can autoscale anything that implements the Kubernetes [scale subresource](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/autoscaling/horizontal-pod-autoscaler.md#scale-subresource). +Karpenter is a metrics-driven autoscaler built for Kubernetes and can run in any Kubernetes cluster anywhere. It's performant, extensible, and can autoscale anything that implements the Kubernetes [scale subresource](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/autoscaling/horizontal-pod-autoscaler.md#scale-subresource). + +This is an early stage, experimental project built with ❤️ and is available as a **developer preview**. We're excited you are here - jump in, let us know what you think. We welcome contributions. ## Getting Started -We will learn about Karpenter's APIs, look at some sample configurations, and install Karpenter's Controller. +We will learn about Karpenter's APIs, look at some sample configurations, and install Karpenter's Controller. Alternatively, you can dive right into the [demo](https://github.com/ellistarn/karpenter-aws-demo). ### APIs Karpenter defines three custom resources to configure autoscaling behavior. -**[HorizontalAutoscalers](./pkg/apis/autoscaling/v1alpha1/horizontalautoscaler.go)** define your autoscaling policy. It's modeled closely after the [HoriontalPodAutoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), but has been generalized to support autoscaling for arbitrary resources. HorizontalAutoscalers periodically query metrics configured by `spec.metrics`, compute an autoscaling decision controlled by `spec.behavior`, and adjust the replicas of their `spec.scaleTargetRef`. Unlike the HPA, Karpenter's HorizontalAutoscalers integrate directly with Prometheus and can use any [promql](https://prometheus.io/docs/prometheus/latest/querying/basics/) response of type "instant vector" in their calculations. +**[HorizontalAutoscalers](./pkg/apis/autoscaling/v1alpha1/horizontalautoscaler.go)** define your autoscaling policy. It's modeled closely after the [HorizontalPodAutoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/), but has been generalized to support autoscaling for arbitrary resources. HorizontalAutoscalers periodically query metrics configured by `spec.metrics`, compute an autoscaling decision controlled by `spec.behavior`, and adjust the replicas of their `spec.scaleTargetRef`. Unlike the HPA, Karpenter's HorizontalAutoscalers integrate directly with Prometheus and can use any [promql](https://prometheus.io/docs/prometheus/latest/querying/basics/) response of type "instant vector" in their calculations. **[MetricsProducers](./pkg/apis/autoscaling/v1alpha1/metricsproducer.go)** generate Prometheus metrics for commonly used autoscaling use cases. They periodically calculate a metric based on their configuration and expose it at a metrics endpoint that can be scraped by Prometheus. If you already have metrics you wish to use for autoscaling available in Prometheus, it is not necessary to define a Metrics Producer. -**[ScalableNodeGroups](./pkg/apis/autoscaling/v1alpha1/scalablenodegroup.go)** provide a minimal way to point a HorizontalAutoscaler's `scaleTargetRef` to a Cloud Provider's Node Group API. Kubernetes core does not define an abstraction for Node Group. Instead, Cloud Providers typically expose non-Kubernetes Node Group APIs. ScalableNodeGroups are a shim in front of these APIs that are limited to `spec.replicas` and `status.replicas`. It is not a replcement or wrapper for these APIs. If you're using a solution that provides a Kubernetes API (e.g. [Kops](https://github.com/kubernetes/kops) or [Cluster API](https://github.com/kubernetes-sigs/cluster-api)), you can point the HorizontalAutoscaler's `scaleTargetRef` to these resources instead of a ScalableNodeGroup. +**[ScalableNodeGroups](./pkg/apis/autoscaling/v1alpha1/scalablenodegroup.go)** provide a minimal way to point a HorizontalAutoscaler's `scaleTargetRef` to a Cloud Provider's Node Group API. Kubernetes core does not define an abstraction for Node Group. Instead, Cloud Providers typically expose non-Kubernetes Node Group APIs. ScalableNodeGroups are a shim in front of these APIs that are limited to `spec.replicas` and `status.replicas`. It is not a replacement or wrapper for these APIs. If you're using a solution that provides a Kubernetes API (e.g. [Kops](https://github.com/kubernetes/kops) or [Cluster API](https://github.com/kubernetes-sigs/cluster-api)), you can point the HorizontalAutoscaler's `scaleTargetRef` to these resources instead of a ScalableNodeGroup. [Learn more](./docs) about the different ways to configure Karpenter's resources. @@ -21,14 +22,32 @@ Karpenter defines three custom resources to configure autoscaling behavior. Follow the setup recommendations of your cloud provider. - [AWS](./docs/aws/README.md#installation) -Then install the controller. +### Quick Install - Controller + Dependencies ``` -CLOUD_PROVIDER=aws -VERSION=v0.1.0 -kubectl apply -f https://raw.githubusercontent.com/awslabs/karpenter/master/releases/${CLOUD_PROVIDER}/${VERSION}.yaml +sh -c "$(curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/v0.1.2/hack/quick-install.sh)" ``` -# Docs +### Kubectl - Standalone +``` +kubectl apply -f https://raw.githubusercontent.com/awslabs/karpenter/v0.1.2/releases/aws/manifest.yaml +``` + +### Helm - Standalone +``` +helm repo add karpenter https://awslabs.github.io/karpenter/charts +helm install karpenter karpenter/karpenter +``` + +## Docs - [Examples](./docs/examples) +- [Working Group](./docs/working-group) - [Developer Guide](./docs/DEVELOPER_GUIDE.md) -- [Design](./docs/DESIGN.md) +- [Design](./docs/designs/DESIGN.md) +- [FAQs](./docs/FAQs.md) +- [Contributing](./docs/CONTRIBUTING.md) + +## Terms +Karpenter is an early stage, experimental project that is currently maintained by AWS and available as a preview. We request that you do not use Karpenter for production workloads at this time. See details in our [terms](./docs/TERMS.md). + +## License +This project is licensed under the Apache-2.0 License. diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 30404ce4c546..000000000000 --- a/ROADMAP.md +++ /dev/null @@ -1 +0,0 @@ -TODO \ No newline at end of file diff --git a/charts/index.yaml b/charts/index.yaml new file mode 100644 index 000000000000..760bcb912943 --- /dev/null +++ b/charts/index.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +entries: + karpenter: + - apiVersion: v2 + created: "2021-02-10T11:12:45.601847-08:00" + description: A Helm chart for https://github.com/awslabs/karpenter/. + digest: 3a64d3c51f5a706df905e49bdd7fb767d87658540864c4b97ffb959c3b15c8dd + name: karpenter + type: application + urls: + - karpenter-v0.1.2.tgz + version: v0.1.2 + - apiVersion: v2 + created: "2021-02-10T11:12:45.601157-08:00" + description: A Helm chart for https://github.com/awslabs/karpenter/. + digest: 39685c8cbe9a757ca48721aed08b49111fef18bc2a9f67d3223f19d0706f09f7 + name: karpenter + type: application + urls: + - karpenter-v0.1.1.tgz + version: v0.1.1 +generated: "2021-02-10T11:12:45.599256-08:00" diff --git a/charts/karpenter-v0.1.1.tgz b/charts/karpenter-v0.1.1.tgz new file mode 100644 index 000000000000..b2472a7dc72c Binary files /dev/null and b/charts/karpenter-v0.1.1.tgz differ diff --git a/charts/karpenter-v0.1.2.tgz b/charts/karpenter-v0.1.2.tgz new file mode 100644 index 000000000000..4db202b1f933 Binary files /dev/null and b/charts/karpenter-v0.1.2.tgz differ diff --git a/charts/karpenter/Chart.lock b/charts/karpenter/Chart.lock new file mode 100644 index 000000000000..e7d712ee29ec --- /dev/null +++ b/charts/karpenter/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: cert-manager + repository: https://charts.jetstack.io + version: v1.1.0 +- name: kube-prometheus-stack + repository: https://prometheus-community.github.io/helm-charts + version: 12.3.0 +digest: sha256:5595919ac269b4105dd65d20eb27cb271b8976c1d10903e0b504d349df30f017 +generated: "2020-12-02T11:48:25.741819-08:00" diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml new file mode 100644 index 000000000000..ba862d11215a --- /dev/null +++ b/charts/karpenter/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: karpenter +description: A Helm chart for https://github.com/awslabs/karpenter/. +type: application +version: v0.1.2 diff --git a/releases/aws/v0.1.0.yaml b/charts/karpenter/templates/manifest.yaml similarity index 99% rename from releases/aws/v0.1.0.yaml rename to charts/karpenter/templates/manifest.yaml index 288034f3d20f..4b331b450a4d 100644 --- a/releases/aws/v0.1.0.yaml +++ b/charts/karpenter/templates/manifest.yaml @@ -759,6 +759,15 @@ rules: - list - patch - watch + - apiGroups: + - '*' + resources: + - '*/scale' + verbs: + - update + - get + - list + - watch - apiGroups: - autoscaling.karpenter.sh resources: @@ -913,7 +922,7 @@ spec: control-plane: karpenter spec: containers: - - image: 197575167141.dkr.ecr.us-west-2.amazonaws.com/karpenter:v0.1.0@sha256:b80ac089c17f15ac37c5f62780c9761e5725463f8a801cb4a4fb69af75c17949 + - image: public.ecr.aws/b6u6q9h4/controller:v0.1.1@sha256:6a5c82cb34bbd6f714145cdfe7c14ac28404a00b56eec9b746ac61eeb3a6d6a8 name: manager ports: - containerPort: 9443 @@ -940,7 +949,7 @@ spec: defaultMode: 420 secretName: webhook-server-cert --- -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Certificate metadata: labels: @@ -956,7 +965,7 @@ spec: name: karpenter-selfsigned-issuer secretName: webhook-server-cert --- -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Issuer metadata: labels: diff --git a/karpenter/main.go b/cmd/controller/main.go similarity index 81% rename from karpenter/main.go rename to cmd/controller/main.go index 5224cd09300d..171ebaedb405 100644 --- a/karpenter/main.go +++ b/cmd/controller/main.go @@ -3,6 +3,7 @@ package main import ( "flag" "fmt" + "github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/reallocator" "github.com/awslabs/karpenter/pkg/apis" "github.com/awslabs/karpenter/pkg/cloudprovider" @@ -12,15 +13,16 @@ import ( "github.com/awslabs/karpenter/pkg/autoscaler" horizontalautoscalerv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/horizontalautoscaler/v1alpha1" metricsproducerv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/metricsproducer/v1alpha1" + "github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/allocator" scalablenodegroupv1alpha1 "github.com/awslabs/karpenter/pkg/controllers/scalablenodegroup/v1alpha1" metricsclients "github.com/awslabs/karpenter/pkg/metrics/clients" "github.com/awslabs/karpenter/pkg/metrics/producers" "github.com/awslabs/karpenter/pkg/utils/log" - "go.uber.org/zap" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" controllerruntime "sigs.k8s.io/controller-runtime" controllerruntimezap "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -52,8 +54,7 @@ func main() { flag.IntVar(&options.MetricsPort, "metrics-port", 8080, "The port the metric endpoint binds to for operating metrics about the controller itself.") flag.Parse() - log.Setup(controllerruntimezap.UseDevMode(options.EnableVerboseLogging)) - + log.Setup(controllerruntimezap.UseDevMode(options.EnableVerboseLogging), controllerruntimezap.ConsoleEncoder()) manager := controllers.NewManagerOrDie(controllerruntime.GetConfigOrDie(), controllerruntime.Options{ LeaderElection: true, LeaderElectionID: "karpenter-leader-election", @@ -67,11 +68,15 @@ func main() { metricsClientFactory := metricsclients.NewFactoryOrDie(options.PrometheusURI) autoscalerFactory := autoscaler.NewFactoryOrDie(metricsClientFactory, manager.GetRESTMapper(), manager.GetConfig()) - if err := manager.Register( + corev1Client, err := corev1.NewForConfig(manager.GetConfig()) + log.PanicIfError(err, "Failed creating kube client") + + err = manager.Register( &horizontalautoscalerv1alpha1.Controller{AutoscalerFactory: autoscalerFactory}, &scalablenodegroupv1alpha1.Controller{CloudProvider: cloudProviderFactory}, &metricsproducerv1alpha1.Controller{ProducerFactory: metricsProducerFactory}, - ).Start(controllerruntime.SetupSignalHandler()); err != nil { - zap.S().Panicf("Unable to start manager, %w", err) - } + allocator.NewController(manager.GetClient(), corev1Client, cloudProviderFactory), + reallocator.NewController(manager.GetClient(), cloudProviderFactory), + ).Start(controllerruntime.SetupSignalHandler()) + log.PanicIfError(err, "Unable to start manager") } diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 40dae13d5557..4bd1868de82c 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -5,6 +5,7 @@ resources: - bases/autoscaling.karpenter.sh_horizontalautoscalers.yaml - bases/autoscaling.karpenter.sh_scalablenodegroups.yaml - bases/autoscaling.karpenter.sh_metricsproducers.yaml +- bases/provisioning.karpenter.sh_provisioners.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 3937c53d0728..9c945a5c86c0 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -22,9 +22,10 @@ spec: labels: control-plane: karpenter spec: + serviceAccountName: karpenter containers: - name: manager - image: ko://github.com/awslabs/karpenter/karpenter + image: ko://github.com/awslabs/karpenter/cmd/controller resources: limits: cpu: 100m diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index c887f9f6f1c7..04b8361bc72d 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -1,4 +1,5 @@ resources: +- serviceaccount.yaml - role.yaml - role_binding.yaml - leader_election_role.yaml diff --git a/config/rbac/leader_election_role_binding.yaml b/config/rbac/leader_election_role_binding.yaml index 3df796616e4e..8f473551ca10 100644 --- a/config/rbac/leader_election_role_binding.yaml +++ b/config/rbac/leader_election_role_binding.yaml @@ -8,5 +8,5 @@ roleRef: name: karpenter-leader-election subjects: - kind: ServiceAccount - name: default + name: karpenter namespace: karpenter diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b89669df25af..a7e71b5ac657 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -1,9 +1,6 @@ - ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: karpenter rules: - apiGroups: @@ -11,25 +8,43 @@ rules: resources: - horizontalautoscalers - horizontalautoscalers/status + - metricsproducers + - metricsproducers/status + - scalablenodegroups + - scalablenodegroups/status + - provisioners + - provisioners/status verbs: - create - delete + - patch - get - list - patch - watch - apiGroups: - - autoscaling.karpenter.sh + - provisioning.karpenter.sh resources: - - metricsproducers - - metricsproducers/status + - provisioners + - provisioners/status verbs: - create - delete + - patch - get - list - patch - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - patch + - update + - watch - apiGroups: - '*' resources: @@ -40,41 +55,32 @@ rules: - list - watch - apiGroups: - - autoscaling.karpenter.sh + - "" resources: - - scalablenodegroups - - scalablenodegroups/status + - nodes + - pods verbs: - - create - - delete - get - list - - patch - watch - apiGroups: - - autoscaling.karpenter.sh + - "" resources: - - scalablenodegroups/scale + - configmaps verbs: - get - - patch + - list + - watch - update - apiGroups: - - coordination.k8s.io + - "" resources: - - leases + - nodes verbs: - create - - get - - patch - - update - - watch - apiGroups: - "" resources: - - nodes - - pods + - pods/binding verbs: - - get - - list - - watch + - create diff --git a/config/rbac/role_binding.yaml b/config/rbac/role_binding.yaml index 00795b327b23..2ee2d420134b 100644 --- a/config/rbac/role_binding.yaml +++ b/config/rbac/role_binding.yaml @@ -8,5 +8,5 @@ roleRef: name: karpenter subjects: - kind: ServiceAccount - name: default + name: karpenter namespace: karpenter diff --git a/config/rbac/serviceaccount.yaml b/config/rbac/serviceaccount.yaml new file mode 100644 index 000000000000..43c884f0cce7 --- /dev/null +++ b/config/rbac/serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: karpenter + namespace: karpenter diff --git a/config/webhook/certificate.yaml b/config/webhook/certificate.yaml index b18fa73d6150..5545a19b9676 100644 --- a/config/webhook/certificate.yaml +++ b/config/webhook/certificate.yaml @@ -2,7 +2,7 @@ # More document can be found at https://docs.cert-manager.io # WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for # breaking changes -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Issuer metadata: name: selfsigned-issuer @@ -10,7 +10,7 @@ metadata: spec: selfSigned: {} --- -apiVersion: cert-manager.io/v1alpha2 +apiVersion: cert-manager.io/v1 kind: Certificate metadata: name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml index 3513703c7ddb..a487978046a7 100644 --- a/config/webhook/kustomization.yaml +++ b/config/webhook/kustomization.yaml @@ -18,7 +18,7 @@ vars: objref: kind: Certificate group: cert-manager.io - version: v1alpha2 + version: v1 name: serving-cert # this name should match the one in certificate.yaml fieldref: fieldpath: metadata.namespace @@ -26,7 +26,7 @@ vars: objref: kind: Certificate group: cert-manager.io - version: v1alpha2 + version: v1 name: serving-cert # this name should match the one in certificate.yaml - name: SERVICE_NAMESPACE # namespace of the service objref: diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..9d9abe6a3c01 --- /dev/null +++ b/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,2 @@ +# Code of Conduct +The Karpenter project follows the [CNCF Community Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 000000000000..fc4f20577295 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,45 @@ +# Contributing Guidelines + +Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional documentation, we greatly value feedback and contributions from our community. + +Please read through this document before submitting any issues or pull requests to ensure we have all the necessary information to effectively respond to your bug report or contribution. + +We follow the [CNCF Community Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). + +## Working Group +Connect with us at our [working group](./working-group). + +## Reporting Bugs/Feature Requests + +We welcome you to use the GitHub issue tracker to report bugs or suggest features. + +When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: + +* A reproducible test case or series of steps +* The version of our code being used +* Any modifications you've made relevant to the bug +* Anything unusual about your environment or deployment + +## Contributing via Pull Requests +Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: + +1. You are working against the latest source on the *master* branch. +2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. +3. You open an issue to discuss any significant work - we would hate for your time to be wasted. + +To send us a pull request, please: + +1. Fork the repository. +2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. +3. Ensure local tests pass. +4. Commit to your fork using clear commit messages. +5. Send us a pull request, answering any default questions in the pull request interface. +6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. + +GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). + +## Code of Conduct +Karpenter has adopted the [CNCF Community Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). + +## Licensing +Karpenter is licensed under [Apache 2.0](./LICENSE.md). diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md index 4feaa1c6a90d..77ce1f916918 100644 --- a/docs/DEVELOPER_GUIDE.md +++ b/docs/DEVELOPER_GUIDE.md @@ -4,12 +4,12 @@ The following tools are required for doing development on Karpenter. -| Package | Version | Install | -| ------------------------------------------------------------------ | -------- | ------------------- | -| [go](https://golang.org/dl/) | v1.15.3+ | | -| [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) | | | -| [helm](https://helm.sh/docs/intro/install/) | | `brew install helm` | -| Other tools | | `make toolchain` | +| Package | Version | Install | +| ------------------------------------------------------------------ | -------- | ---------------------- | +| [go](https://golang.org/dl/) | v1.15.3+ | `brew install go` | +| [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) | | `brew install kubectl` | +| [helm](https://helm.sh/docs/intro/install/) | | `brew install helm` | +| Other tools | | `make toolchain` | ## Developing @@ -18,7 +18,7 @@ The following tools are required for doing development on Karpenter. Based on which environment you are running a Kubernetes cluster, follow the [Environment specific setup](##Environment-specific-setup) for setting up your environment before you continue. Once you have the environment specific settings, to install Karpenter in a Kubernetes cluster run the following commands. ``` -make generate # Create auto-generated YAML files. +make codegen # Create auto-generated YAML files. ./hack/quick-install.sh # Install cluster dependencies and karpenter ./hack/quick-install.sh --delete # Clean everything up ``` @@ -26,7 +26,7 @@ make generate # Create auto-generated YAML files. ### Developer Loop * Make sure dependencies are installed - * Run `make generate` to make sure yaml manifests are generated + * Run `make codegen` to make sure yaml manifests are generated * Run `make toolchain` to install cli tools for building and testing the project * You will need a personal development image repository (e.g. ECR) * Make sure you have valid credentials to your development repository. @@ -34,7 +34,7 @@ make generate # Create auto-generated YAML files. * Your cluster must have permissions to read from the repository * Make sure your cluster doesn't have previous installations of prometheus and cert-manager * Previous installations of our dependencies can interfere with our installation scripts, so to be safe, clear those, then run `./hack/quick-install.sh` -* If running `./hack/quick-install.sh` fails with `Error: Accumulate Target`, run `make generate` successfully, and try again. +* If running `./hack/quick-install.sh` fails with `Error: Accumulate Target`, run `make codegen` successfully, and try again. ### Build and Deploy ``` @@ -49,20 +49,25 @@ make test # E2e correctness tests make battletest # More rigorous tests run in CI environment ``` +### Verbose Logging +```bash +kubectl patch deployment karpenter -n karpenter --type='json' -p='[{"op": "replace", "path": "/spec/template/spec/containers/0/args", "value": ["--verbose"]}]' +``` + ### Debugging Metrics Prometheus -``` +```bash open http://localhost:9090/graph && kubectl port-forward service/prometheus-operated -n karpenter 9090 ``` Karpenter Metrics -``` +```bash open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics-service -n karpenter 8080 ``` ## Environment specific setup ### AWS -Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. +Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. ``` export CLOUD_PROVIDER=aws @@ -72,14 +77,14 @@ For local development on Karpenter you will need a Docker repo which can manage You can use the following command to provision an ECR repository. ``` aws ecr create-repository \ - --repository-name karpenter \ + --repository-name karpenter/controller \ --image-scanning-configuration scanOnPush=true \ - --region ${REGION} + --region ${AWS_DEFAULT_REGION} ``` Once you have your ECR repository provisioned, configure your Docker daemon to authenticate with your newly created repository. ``` -export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${REGION}.amazonaws.com" -aws ecr get-login-password --region ${REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO +export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter" +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO ``` diff --git a/docs/FAQs.md b/docs/FAQs.md new file mode 100644 index 000000000000..50fc875704dc --- /dev/null +++ b/docs/FAQs.md @@ -0,0 +1,58 @@ +# Frequently Asked Questions + +1. **Why should I use Karpenter?** +Karpenter enables Kubernetes users to maximize resource utilization and improve availability for their clusters without requiring them to manually allocate or over-provision resources. Users can choose the metrics they want to drive the amount of compute resources allocated for their cluster, letting them scale their clusters independently, ahead-of, or in-concert with the scale of their applications. Users can configure scaling across multiple compute options and Karpenter offers straightforward and highly customizable options for scaling that are defined with configuration files, so they can be easily shared and implemented across multiple clusters. Karpenter runs as a set of linked components within a Kubernetes cluster, which allows the system to make fast, linear time scaling decisions for any size of cluster. + +2. **How do I start using Karpenter?** +Check out our [Getting Started documentation](/README.md#getting-started) for installation instructions and common usage patterns. + +3. **How does Karpenter compare to the Kubernetes cluster autoscaler?** +The Kubernetes cluster autoscaler reacts to the number of pods running on the cluster and tightly coupled to the type of compute used as well as the number of pods running. This means that cluster autoscaler will not scale a cluster until more pods are running and means that for workloads where you desire to scale on a schedule or over-provision, you need to implement work arounds such as ‘Hollow Pods’ that essentially hack cluster autoscaler. Karpenter is more flexible. You can apply a traditional pending pods metric to some of your node groups, while using different metrics like reserved capacity to scale other node groups. This gives Karpenter the flexibility to implement sophisticated scaling behavior to accommodate a wide variety of use cases. + +4. **Where can I use Karpenter?** +Karpenter works with any Kubernetes cluster running in any environment. You can use Karpenter with Kubernetes clusters in the cloud and on premises. + +5. **How does Karpenter work?** +Karpenter is an open, metrics-driven autoscaling system. There are four logical components: 1/ metrics producer which outputs metrics that can be used to drive scaling, 2/ metrics server which aggregates and stores scaling metric data from each producer, 3/ autoscaler which contains the logic for scaling including metric tracking strategies and scaling policies, and 4/ replica controller which changes the number of desired replicas for a unit of compute. These components are able to be implemented together or flexibly in combination with other systems such as KEDA. See appendix for more information about the Karpenter system architecture. + +6. **Is Karpenter a node autoscaler only? Are there plans to make it extensible for workloads as well?** +At launch, we plan to build replica controllers to drive node scaling for Kubernetes clusters. That said, Karpenter has an open design can be used to scale anything that implements the scale sub-resource. This includes multiple Kubernetes resource controllers and even cloud provider resources that are controlled from the Kuberentes API (such as with [ACK](https://github.com/aws/aws-controllers-k8s)). + +7. **Can I define my own metrics?** +Yes! You can write your own metrics producers for any metric you want to use to scale your cluster. + +8. **How does scaling work in Karpenter?** +Karpenter manages scaling based on the principals of proportional control. This means that Karpenter attempts to maintain a desired number of compute resources in proportion to a scaling metric that you define. Similar to how you set a minimum and maximum temperature for your house’s thermostat, you can set separate scaling rules for these proportions with relation to both scaling up and scaling down. Each scaling rule includes a scaling policy that allows you to define which metric to scale off of and how to scale based on that metric. In Karpenter you can define multiple metrics, and multiple scaling policies and apply these to separate node groups. In this way, Karpenter can be as simple, or as complex, as your use case dictates. If you want to simply scale up all nodes based on the number of items in a queue or inbound connection requests at a load balancer, you can do that. If you want to scale up certain node groups based on CPU utilization and only scale down if traffic to the website drops below a particular threshold, you can do that also. + +9. **Some cloud providers have existing managed scaling systems, such as predictive scaling. Can Karpenter use those?** +Yes. Each component of Karpenter is decoupled so that any can be swapped out to take advantage of existing managed systems. This means that any existing predictive scaling system can be integrated into Karpenter by using that scaling system as a metrics source. Additionally, Karpenter can feed data to these systems by sending them scale decisions as a replica target. At the extreme, this could mean using Karpenter as an API standard and having all functionality fulfilled by other systems. + +10. **What kinds of metric targets can I use to setup scaling policies?** +Karpenter can scale using three types of metric targets: value, average-value, and utilization. Value and utilization let you drive scaling based on the metric signal and existing replicas. Average value lets you track scaling to a desired input metric value and is independent of replicas. You can use these interchangeably with metrics signals and Karpenter scaling policies to drive different scaling behavior. + +11. **Can Karpenter scale from/to 0?** +Because average value metric target type is independent of the number of existing replicas in a cluster, you can use this metric target type to drive scaling decisions when replicas are 0. Furthermore, Karpenter lets you define multiple metrics signals in a single policy, allowing you to scale from zero using an average value metric and then beyond that using a completely different metric. + +12. **Does Karpenter work with multiple types of nodes?** +Karpenter allows you to target specific policies for specific node groups. For example, you can scale one group by one node for every 1k inbound connection requests and another group by two nodes for every 1k inbound connection requests. When Karpenter sees the inbound connections increase, it will request the appropriate node group to scale by the relative amount you define. This pattern works with different sizes of node groups, but also can be extended to different types of nodes like ARM and Spot where you need the control over which applications and scenarios scale these groups. Karpenter lets you directly define the relationship between the scaling metric and what is scaled. This gives you fine-grained control in how to scale different types of nodes across your cluster and still allows you to apply globally computed metrics to a set of node groups. + +13. **Does Karpenter work with EC2 Spot?** +Yes. Karpenter allows you to specify fine grained policies on any node group. You can configure multiple node groups to scale with different or the same metrics, depending on your use case. One way to do this with spot is to configure a `capacityReservation` metric for two node groups with spot instances of different instance types. As the scheduler fills the nodes with incoming pods, the node groups will scale out. If one of the instances types becomes unavailable, the other node group will continue to scale. + +14. **Does Karpenter respect pod disruption budgets?** +Karpenter does not include an integration to Kubernetes lifecycle hooks in order to drain nodes during scaling. However, Karpenter does allow you to connect resources like an EKS managed node group (MNG) to the replica controller. Resources such as MNG have existing integrations to Kubernetes lifecycle events to ensure graceful scale down events. + +15. **How does Karpenter work with Prometheus?** +Karpenter uses promql queries for its HorizontalAutoscaler. Any metrics available in Prometheus can be used to autoscale a resource. For example, you can use Karpenter's MetricsProducer resource, kube-state-metrics, or any custom code that exposes metrics to Prometheus in your scaling policies. + +16. **Metrics-driven open source autoscaling systems like HPA and KEDA exist today. How is Karpenter different?** +Systems including HPA are similar to Karpenter, but designed to manage pod scaling with the assumption that a reactive node scaling system will ensure enough compute is available for the cluster. Karpenter takes a very similar approach to these exiting metrics-driven systems, and uses many of the same principals, applying them to node scaling. + +17. **Cluster Autoscaler has worked just fine for my clusters, why should I use Karpenter?** +Cluster autoscaler works well for a number of common use cases. However, some use cases such as scheduled scaling or batch processing, you have to do a lot of extra work or accept the performance and resource inefficiencies created by the architecture of Cluster Autoscaler reacting to pending pods. If you have a mix of use cases in your organization, this means that some users have a fundamentally different architecture and approach to scaling their clusters based on what they are doing. Karpenter allows standardization in how scaling works across your entire organization and for all use cases. Its flexibility lets you optimize your cluster scaling to meet any Kubernetes application use case while reducing implementation complexities and differences that cause delays and take significant extra work on behalf of some teams. + +18. **Metrics will be polled periodically to calculate the desired replicas. Is it possible to configure the polling period?** +The default polling period is 10 seconds, though the user can configure this in their HorizontalAutoscaler policy. + +19. **Is this just an AWS project?** +This is an AWS initiated project, but we intend our working group to grow to members across the Kubernetes community. We welcome and encourage anyone to join us! See [contributing](./CONTRIBUTING.md). diff --git a/docs/README.md b/docs/README.md index 1ed7913c0b98..c8baeb14ba0a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1622,5 +1622,5 @@ map[string]string

Generated with gen-crd-api-reference-docs -on git commit e85ad5d. +on git commit 52b7290.

diff --git a/docs/TERMS.md b/docs/TERMS.md new file mode 100644 index 000000000000..2d27eb8b9f7c --- /dev/null +++ b/docs/TERMS.md @@ -0,0 +1,4 @@ +# Terms of use +Karpenter is an early stage, experimental project that is currently maintained by AWS and available as a preview. +Use of Karpenter in preview on AWS is subject to the terms and conditions contained in the AWS Service Terms, particularly the Beta Service Participation Service Terms, located at https://aws.amazon.com/service-terms (the “Beta Terms”). +In addition to the Beta Terms, the following terms and conditions apply to your use of Karpenter: Karpenter is not intended for production workloads. You are responsible for fees incurred for other AWS Services that you use in connection with Karpenter. Standard pricing will apply for your use of those AWS Services. diff --git a/docs/aws/README.md b/docs/aws/README.md index 5de546bdb93d..0455b8b1aadc 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -3,77 +3,48 @@ ## Installation Karpenter's pod requires AWS Credentials to read or modify AWS resources in your account. We recommend using IRSA (IAM Roles for Service Accounts) to manage these permissions. -``` +```bash AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) CLUSTER_NAME= REGION= ``` -### Create the Karpenter IAM Policy -This command will create an IAM Policy with access to all of the resources for all of Karpenter's features. For increased security, you may wish to reduce the permissions according to your use case. -``` -aws iam create-policy --policy-name Karpenter --policy-document "$(cat <<-EOM -{ - "Version": "2012-10-17", - "Statement": [ - { - "Action": [ - "eks:DescribeNodegroup", - "eks:UpdateNodegroupConfig" - ], - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "autoscaling:DescribeAutoScalingGroups", - "autoscaling:UpdateAutoScalingGroup" - ], - "Effect": "Allow", - "Resource": "*" - }, - { - "Action": [ - "sqs:GetQueueAttributes", - "sqs:GetQueueUrl" - ], - "Effect": "Allow", - "Resource": "*" - } - ] -} -EOM -)" +### Create Karpenter IAM Resources +This command will create IAM resources used by Karpenter. For production use, please review and restrict these permissions as necessary. +```bash +// TODO, point to github raw uri +aws cloudformation deploy \ + --stack-name Karpenter \ + --template-file ./docs/aws/karpenter.cloudformation.yaml \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides OpenIDConnectIdentityProvider=$(aws eks describe-cluster --name ${CLUSTER_NAME} | jq -r ".cluster.identity.oidc.issuer" | cut -c9-) ``` -### Associate the IAM Role with your Kubernetes Service Account -These commands will associate the AWS IAM Policy you created above with the Kubernetes Service Account used by Karpenter. -``` +### Enable IRSA +Enables IRSA for your cluster. This command is idempotent, but only needs to be executed once per cluster. +```bash eksctl utils associate-iam-oidc-provider \ --region ${REGION} \ --cluster ${CLUSTER_NAME} \ --approve - -eksctl create iamserviceaccount --cluster ${CLUSTER_NAME} \ ---name default \ ---namespace karpenter \ ---attach-policy-arn "arn:aws:iam::${AWS_ACCOUNT_ID}:policy/Karpenter" \ ---override-existing-serviceaccounts \ ---approve ``` -### Verify the Permissions -You should see an annotation with key eks.amazonaws.com/role-arn -``` -kubectl get serviceaccount default -n karpenter -ojsonpath="{.metadata.annotations}" +### Attach the Permissions +```bash +kubectl patch serviceaccount karpenter -n karpenter --patch "$(cat <<-EOM +metadata: + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole +EOM +)" ``` If you've already installed the Karpenter controller, you'll need to restart the pod to load the credentials. -``` +```bash kubectl delete pods -n karpenter -l control-plane=karpenter ``` ### Cleanup -``` -eksctl delete iamserviceaccount --cluster ${CLUSTER_NAME} --name default --namespace karpenter -aws iam delete-policy --policy-arn arn:aws:iam::${AWS_ACCOUNT_ID}:policy/Karpenter +```bash +aws cloudformation delete-stack --stack-name Karpenter +aws ec2 describe-launch-templates | jq -r ".LaunchTemplates[].LaunchTemplateName" | grep Karpenter | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} ``` diff --git a/docs/aws/karpenter.cloudformation.yaml b/docs/aws/karpenter.cloudformation.yaml new file mode 100644 index 000000000000..be7003dcaabf --- /dev/null +++ b/docs/aws/karpenter.cloudformation.yaml @@ -0,0 +1,80 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: Resources used by https://github.com/awslabs/karpenter +Parameters: + OpenIDConnectIdentityProvider: + Type: String + Description: "Example oidc.eks.us-west-2.amazonaws.com/id/1234567890" +Resources: + KarpenterControllerRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: KarpenterControllerRole + Path: / + AssumeRolePolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OpenIDConnectIdentityProvider}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "${OpenIDConnectIdentityProvider}:aud": "sts.${AWS::URLSuffix}", + "${OpenIDConnectIdentityProvider}:sub": "system:serviceaccount:karpenter:karpenter" + } + } + }] + } + KarpenterControllerPolicy: + Type: "AWS::IAM::Policy" + Properties: + PolicyName: Karpenter + Roles: + - + Ref: "KarpenterControllerRole" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Resource: "*" + Action: + # Write Operations + - "ec2:CreateLaunchTemplate" + - "ec2:CreateFleet" + - "ec2:RunInstances" + - "ec2:CreateTags" + - "iam:PassRole" + # Read Operations + - "ec2:DescribeLaunchTemplates" + - "ec2:DescribeInstances" + - "ec2:DescribeSecurityGroups" + - "ec2:DescribeSubnets" + - "iam:GetInstanceProfile" + KarpenterNodeInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + InstanceProfileName: "KarpenterNodeInstanceProfile" + Path: "/" + Roles: + - Ref: "KarpenterNodeInstanceRole" + KarpenterNodeInstanceRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "KarpenterNodeRole" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" diff --git a/docs/DESIGN.md b/docs/designs/DESIGN.md similarity index 84% rename from docs/DESIGN.md rename to docs/designs/DESIGN.md index 156efea7f78c..9d4d0f89b59d 100644 --- a/docs/DESIGN.md +++ b/docs/designs/DESIGN.md @@ -1,10 +1,10 @@ # Metrics Driven Autoscaling -Node Autoscaling (a.k.a. Cluster Autoscaling) is the process of continually adding and removing a cluster’s nodes to meet the resource demands of its pods. As customers scale to increasingly large clusters, autoscaling becomes necessary for both practicality and cost reasons. While overprovisioning is a viable approach at smaller scales, it becomes prohibitively expensive as organizations grow. In response to increasing infrastructure costs, some customers create manual processes to scale node groups, but this approach yields inefficient resource utilization and is error prone. Node autoscaling replaces these manual processes with automation. +Node Autoscaling (a.k.a. Cluster Autoscaling) is the process of continually adding and removing a cluster’s nodes to meet the resource demands of its pods. As users scale to increasingly large clusters, autoscaling becomes necessary for both practicality and cost reasons. While overprovisioning is a viable approach at smaller scales, it becomes prohibitively expensive as organizations grow. In response to increasing infrastructure costs, some users create manual processes to scale node groups, but this approach yields inefficient resource utilization and is error prone. Node autoscaling replaces these manual processes with automation. ## Overview -Metrics driven autoscaling architectures are widespread in the Kubernetes ecosystem, including Kubernetes Horizontal Pod Autoscaler, Knative, and KEDA. Public clouds also use metrics driven architectures, such as [EC2 Autoscaling](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-scaling-target-tracking.html) and [ECS Autoscaling](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-auto-scaling.html). This approach has been attempted for node autoscaling in kubernetes by project [Cerebral](https://github.com/containership/cerebral), although it is [no longer actively maintained](https://github.com/containership/cerebral/issues/124#issuecomment-679363530). Existing node autoscaling solutions suffer from complexity, inflexibility, performance, and scalability issues. The extensibility of these systems is limited by both configuration mechanism as well as fundamental architectural constraints, preventing users from applying multiple scaling policies to their cluster or using arbitrary signals to control scaling actions. +Metrics driven autoscaling architectures are widespread in the Kubernetes ecosystem, including Kubernetes Horizontal Pod Autoscaler, Knative, and KEDA. Public clouds also use metrics driven architectures, such as [EC2 Autoscaling](https://docs.aws.amazon.com/autoscaling/ec2/userguide/as-scaling-target-tracking.html) and [ECS Autoscaling](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/service-auto-scaling.html). This approach has been attempted for node autoscaling in Kubernetes by project [Cerebral](https://github.com/containership/cerebral), although it is [no longer actively maintained](https://github.com/containership/cerebral/issues/124#issuecomment-679363530). Existing node autoscaling solutions suffer from complexity, inflexibility, performance, and scalability issues. The extensibility of these systems is limited by both configuration mechanism as well as fundamental architectural constraints, preventing users from applying multiple scaling policies to their cluster or using arbitrary signals to control scaling actions. We will first discuss metrics driven autoscaling in the abstract, before applying the techniques to the domain of node autoscaling. We will also evaluate the landscape of existing Kubernetes ecosystem projects that will be either used to rapidly implement this approach or be aligned with in the long term. @@ -18,21 +18,21 @@ Many aspects of this design contain large subproblems that are beyond the scope * Provide straightforward tradeoffs for Scalability, Performance, Availability, and Cost. * Maximize the compatibility with existing solutions within the Kubernetes ecosystem. -## Critical Customer Journeys +## Critical User Journeys -* As a customer, I am able to scale up or down on a single or combination of multiple signals. +* As a user, I am able to scale up or down on a single or combination of multiple signals. * e.g. Capacity Reservations, Scheduled Capacity, Pending Pods, Queue Length -* As a customer, I am able to author my own custom metrics to plug into the autoscaling architecture. -* As a customer, I am able to autoscale multiple node group implementations from different providers. +* As a user, I am able to author my own custom metrics to plug into the autoscaling architecture. +* As a user, I am able to autoscale multiple node group implementations from different providers. * e.g. EC2 Autoscaling Groups, EKS Managed Node Groups, Kops Instance Groups, etc. -* As a customer, I am able to provision and autoscale capacity in the same cluster with disjoint scheduling properties. +* As a user, I am able to provision and autoscale capacity in the same cluster with disjoint scheduling properties. * e.g. GPUs, HPC, Spot Instances, node labels and taints. ## Metrics Driven Autoscaling Architecture Metrics Driven Autoscaling is broken down into four logical components. -![](./images/overview.jpeg) +![](../images/overview.jpeg) These components are able to be implemented flexibly, either combined into a single system (e.g. Kubernetes Cluster Autoscaler), using one system per component (e.g. Horizontal Pod Autoscaler), or some combination (e.g. KEDA, which implements a Metrics Producer/Metrics Server; Knative, which implements a Metrics Producer/Metrics Server/Autoscaler) @@ -46,11 +46,11 @@ Once generated, metrics must be stored somewhere. Some metrics server implementa ### 3. Autoscaler -Metrics values are periodically polled and used to calculate desiredReplicas for the autoscaled resource. The autoscaler contains a generic, black-box autoscaling function that can be parameterized by customers. +Metrics values are periodically polled and used to calculate desiredReplicas for the autoscaled resource. The autoscaler contains a generic, black-box autoscaling function that can be parameterized by users. `replicas = f(currentReplicas, currentMetricValue, desiredMetricValue, params...)` -This implementation of the function can be a proportional controller ([see HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details)), a [PID controller](https://en.wikipedia.org/wiki/PID_controller), a [predictive controller](https://netflixtechblog.com/scryer-netflixs-predictive-auto-scaling-engine-part-2-bb9c4f9b9385), or something else entirely. These functions are generic such that customers should be able experiment with different autoscaling functions using the same underlying metrics. Input metrics can be any signal. For example, customers could use a raw signal or transform their metric with some (e.g. step) function before it is input into the autoscaler. +This implementation of the function can be a proportional controller ([see HPA](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details)), a [PID controller](https://en.wikipedia.org/wiki/PID_controller), a [predictive controller](https://netflixtechblog.com/scryer-netflixs-predictive-auto-scaling-engine-part-2-bb9c4f9b9385), or something else entirely. These functions are generic such that users should be able experiment with different autoscaling functions using the same underlying metrics. Input metrics can be any signal. For example, users could use a raw signal or transform their metric with some (e.g. step) function before it is input into the autoscaler. ### 4. Replica Controller @@ -58,7 +58,7 @@ The replica controller is responsible for the actual actuation of desiredReplica ## Node Autoscaling -For node autoscaling, configurations are applied to node groups, drawing parallels to how pod autoscaling applies to deployments. This deviates from existing solutions like the Kubernetes Cluster Autoscaler or Escalator, which globally consider all node groups in their scaling decisions. It gives customers the flexibility to configure policies on different capacity types, but does not limit customers from applying globally computed metrics to a set of node groups. +For node autoscaling, configurations are applied to node groups, drawing parallels to how pod autoscaling applies to deployments. This deviates from existing solutions like the Kubernetes Cluster Autoscaler or Escalator, which globally consider all node groups in their scaling decisions. It gives users the flexibility to configure policies on different capacity types, but does not limit users from applying globally computed metrics to a set of node groups. Assuming that a metrics driven approach results in significantly improved performance, flexibility, and functionality, the following questions emerge. @@ -74,7 +74,7 @@ While we will strive to leverage existing systems where it makes sense, there wi Karpenter is a metrics driven node autoscaler. It’s implemented as a Kubernetes controller and defines three custom resources: MetricsProducer, HorizontalAutoscaler, and ScalableNodeGroup. It aligns its interfaces as closely as possible to the Horizontal Pod Autoscaler’s interface, with a long term goal of providing a universal HorizontalAutoscaler definition in upstream Kubernetes. It takes a dependency on [Prometheus](https://prometheus.io/) and provides out-of-the-box support for commonly used metrics producers for Capacity Reservations, Scheduled Capacity, Pending Pods, and Queue Length. -![](./images/design.jpeg) +![](../images/design.jpeg) Before deep diving the design questions, we’ll cover some examples to see how Karpenter works for some common cases. @@ -95,7 +95,7 @@ spec: id: arn:aws:sqs:us-west-2:1234567890:alice-ml-training-queue ``` -Her “ml-training-queue” configures Karpenter to periodically monitor queue metrics, such as the length of her AWS SQS Queue. The monitoring process has a Prometheus metrics endpoint at /metrics that returns the a set of metrics about the queue, including queue length. Alice has Prometheus Server installed in her cluster, which dynamically discovers and periodically scrapes the queue length from the metrics producer and stores it in a timeseries database. Alice queries this data manually using karpenter:metrics_producer:queue-length{name="ml-training-queue", namespace="alice"} to make sure that everything is working smoothly. +Her “ml-training-queue” configures Karpenter to periodically monitor queue metrics, such as the length of her AWS SQS Queue. The monitoring process has a Prometheus metrics endpoint at /metrics that returns the a set of metrics about the queue, including queue length. Alice has a Prometheus Server installed in her cluster, which dynamically discovers and periodically scrapes the queue length from the metrics producer and stores it in a timeseries database. Alice queries this data manually using karpenter:metrics_producer:queue-length{name="ml-training-queue", namespace="alice"} to make sure that everything is working smoothly. ``` apiVersion: karpenter.sh/v1alpha1 @@ -133,7 +133,7 @@ Alice enqueues 2400 tasks into her queue, Karpenter’s PID algorithm quickly co ### Example: Reserving Capacity -Bob is a coworker of Alice and their teams share the same cluster. His team hosts a set of microservices for a product that is gaining new customers every day. Customers choose Bob’s product since it has much lower latency than alternatives. Bob is working with a limited infrastructure budget and needs to minimize costs while making sure his applications are scaling with customer demands. He configures a pod autoscaler for each microservice, which will scale up to maintain low latency as long as capacity is available. Bob’s nodes have 16 cores and 20gb of memory each, and each microservice pod has a resource request of 1 core and 1 gb memory. He is willing to pay for 40% capacity overhead to minimize the chance that a pod will be unschedulable due to unavailable capacity. +Bob is a coworker of Alice and their teams share the same cluster. His team hosts a set of microservices for a product that is gaining new customers every day. Users choose Bob’s product since it has much lower latency than alternatives. Bob is working with a limited infrastructure budget and needs to minimize costs while making sure his applications are scaling with user demands. He configures a pod autoscaler for each microservice, which will scale up to maintain low latency as long as capacity is available. Bob’s nodes have 16 cores and 20gb of memory each, and each microservice pod has a resource request of 1 core and 1 gb memory. He is willing to pay for 40% capacity overhead to minimize the chance that a pod will be unschedulable due to unavailable capacity. He creates two Karpenter resources and applies them with kubectl apply: ``` @@ -180,24 +180,24 @@ Bob starts out with 9 pods, resulting in (9/16)=.5625 CPU and (9/20)=.45 memory Which metrics technology stack should Karpenter leverage? -Kubernetes has an established landscape for metrics-driven autoscaling. This work was [motivated by and evolved alongside](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) the Horizontal Pod Autoscaler. The [Kubernetes monitoring architecture](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-metrics-pipeline/) defines three metrics APIs that are implemented as [kubernetes aggregated APIs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/). +Kubernetes has an established landscape for metrics-driven autoscaling. This work was [motivated by and evolved alongside](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md) the Horizontal Pod Autoscaler. The [Kubernetes monitoring architecture](https://kubernetes.io/docs/tasks/debug-application-cluster/resource-metrics-pipeline/) defines three metrics APIs that are implemented as [Kubernetes aggregated APIs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/apiserver-aggregation/). * metrics.k8s.io provides metrics for pod and node resources like cpu and memory. -* custom.metrics.k8s.io provides metrics for arbitrary kubernetes objects like an Ingress’s qps. +* custom.metrics.k8s.io provides metrics for arbitrary Kubernetes objects like an Ingress’s qps. * external.metrics.k8s.io. provides metrics from systems outside of the cluster. Each API must be implemented by a service in the cluster. Implementations include the [metrics-server](https://github.com/kubernetes-sigs/metrics-server) for metrics.k8s.io, [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for custom.metrics.k8s.io, and [KEDA](https://github.com/kedacore/keda) for external.metrics.k8s.io. For example, here’s how the Horizontal Pod Autoscaler uses [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) and custom.metrics.k8s.io. -![](./images/hpa.png) +![](../images/hpa.png) Source: https://towardsdatascience.com/kubernetes-hpa-with-custom-metrics-from-prometheus-9ffc201991e -The metrics API is an attractive dependency for several reasons. It uses Kubernetes API semantics, bringing popular Kubernetes features (e.g. kubectl, API standardization) to the domain of metrics. It also enables customers to control access using RBAC, though this isn’t hugely compelling as autoscalers typically operate globally on the cluster and have full permissions to the metrics API (see HPA). +The metrics API is an attractive dependency for several reasons. It uses Kubernetes API semantics, bringing popular Kubernetes features (e.g. kubectl, API standardization) to the domain of metrics. It also enables users to control access using RBAC, though this isn’t hugely compelling as autoscalers typically operate globally on the cluster and have full permissions to the metrics API (see HPA). The metrics API has drawbacks. Each API can only have [one implementation](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md), which creates compatibility challenges when trying to deploy multiple applications that attempt to implement the same metrics API. It’s tempting to make Karpenter an external metrics API implementation by leveraging [existing open source libraries](https://github.com/kubernetes-sigs/custom-metrics-apiserver). In fact, this is exactly how KEDA (a popular metrics driven pod autoscaler) implements its metrics stack. However, this approach would mean that Karpenter could not be deployed to any cluster that uses KEDA or any other external metrics API implementation. This “single-implementation“ design decision has led other autoscaling solutions like [Knative Serving](https://github.com/knative/serving) to avoid dependency on these [metrics APIs](https://github.com/knative/serving/issues/9087#issuecomment-675138178). -Given this constraint, something generic and universal could implement the metrics APIs and then allow systems like Karpenter to feed metrics into it. The [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) is a community solution which attempts to be this solution and uses Prometheus as an intermediary, but the adapter must be [manually configured for each metric it exposes](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md). This is a nontrivial customer burden that requires deep knowledge of Prometheus, the k8s-prometheus-adapter, the metrics producer, and Kubernetes metrics APIs. We could explore building a convention for naming external metrics such that the adapter can automatically translate metrics API resources into their external counterparts, removing the need for additional configuration. This [used to be supported](https://github.com/DirectXMan12/k8s-prometheus-adapter#presentation) by k8s-prometheus adapter, but was deprecated in favor of explicit configuration and a configuration generator. +Given this constraint, something generic and universal could implement the metrics APIs and then allow systems like Karpenter to feed metrics into it. The [k8s-prometheus-adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) is a community solution which attempts to be this solution and uses Prometheus as an intermediary, but the adapter must be [manually configured for each metric it exposes](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md). This is a nontrivial user burden that requires deep knowledge of Prometheus, the k8s-prometheus-adapter, the metrics producer, and Kubernetes metrics APIs. We could explore building a convention for naming external metrics such that the adapter can automatically translate metrics API resources into their external counterparts, removing the need for additional configuration. This [used to be supported](https://github.com/DirectXMan12/k8s-prometheus-adapter#presentation) by k8s-prometheus adapter, but was deprecated in favor of explicit configuration and a configuration generator. It’s also possible to closely align with KEDA and share an external metrics API server for both pod and node autoscaling. This introduces a project alignment challenge, but it is not insurmountable. Even if this could work, it’s not a perfect solution, as there will continue to be compatibility issues with other Kubernetes metrics API implementations. @@ -207,7 +207,7 @@ Prometheus is ubiquitous throughout the Kubernetes ecosystem. It was the second There are a few drawbacks to diverging from the existing Kubernetes Metrics APIs. It forces divergence from the Horizontal Pod Autoscaler’s architecture (see next section), which may cause alignment challenges in the future. Kubernetes metrics APIs also come with RBAC support, but Prometheus does not have per-metric authorization. There are also tools like kubectl top which rely on the metrics API, but this command is specific to pod metrics and not useful for metrics used by node autoscaling. -Direct Prometheus integration appears to be the best option. It avoids compatibility issues with other metrics providers. Generic metrics API adapters like k8s-prometheus-adapter create a domain knowledge and configuration burden for customers. This decision has cascading effects to the rest of the design and should be considered very carefully. However, it is a two way door. The API can be flexible to arbitrary metrics stacks, including non-Prometheus alternatives. Prometheus will be considered a soft dependency; it will serve as our reference metrics implementation for Karpenter’s MVP. +Direct Prometheus integration appears to be the best option. It avoids compatibility issues with other metrics providers. Generic metrics API adapters like k8s-prometheus-adapter create a domain knowledge and configuration burden for users. This decision has cascading effects to the rest of the design and should be considered very carefully. However, it is a two way door. The API can be flexible to arbitrary metrics stacks, including non-Prometheus alternatives. Prometheus will be considered a soft dependency; it will serve as our reference metrics implementation for Karpenter’s MVP. ### Alignment with the Horizontal Pod Autoscaler API @@ -215,15 +215,15 @@ Is it possible or worthwhile to align with the Horizontal Pod Autoscaler? The Horizontal Pod Autoscaler (HPA) is a metrics driven pod autoscaling solution in upstream Kubernetes. It’s maintained by SIG Autoscaling and is the canonical solution for the Kubernetes community. Its API has undergone significant changes as Kubernetes has evolved. It initially provided support for scaling a deployment against the average CPU of its Pods, but has since expanded its flexibility in the [v2beta2 API](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec) to support arbitrary resource targets and custom metrics. It can target any Kubernetes resource that implements the [scale subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#subresources). Today, the existing HPA API is even able to target a Kubernetes resource representing a node group; the only gap is to implement metrics for the domain of node autoscaling. -Unified autoscaling is a powerful concept, as it means that the same implementation can be shared for all autoscaled resources within a cluster. We want to avoid forcing premature alignment, but as long as it doesn’t compromise the design, there is value in keeping these interfaces as similar as possible. Customers need only learn a single architecture for autoscaling, reducing complexity and cognitive load. +Unified autoscaling is a powerful concept, as it means that the same implementation can be shared for all autoscaled resources within a cluster. We want to avoid forcing premature alignment, but as long as it doesn’t compromise the design, there is value in keeping these interfaces as similar as possible. Users need only learn a single architecture for autoscaling, reducing complexity and cognitive load. -There are a couple drawbacks to using the HPA’s API directly. The most obvious is the name, which would be more aptly called HorizontalAutoscaler. Most of its abstractions extend cleanly to Node Groups (e.g. [ScaleTargetRef](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [MetricTarget](https://godoc.org/k8s.io/api/autoscaling/v2beta2#MetricTarget), [ScalingPolicy](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingPolicy), [MinReplicas](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [MaxReplicas](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [Behavior](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerBehavior), StabilizationWindowSeconds (https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingRules)). Others require slight adjustments (e.g. [ScalingPolicyType](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingPolicyType) needs to be tweaked to refer to “replicas” instead of “pods”). However, [MetricSpec](https://godoc.org/k8s.io/api/autoscaling/v2beta2#MetricSpec) is specific to pods and requires changes if relied upon. MetricsSpec has four subfields corresponding to different metrics sources. [ResourceMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ResourceMetricSource), which uses the [Resource Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) and provides CPU and memory for pods and nodes. [PodsMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#PodsMetricSource), which is syntactic sugar for [ObjectMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ObjectMetricSource), each of which each retrieve metrics from the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md). [ExternalMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ExternalMetricSource), which uses the [External Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/external-metrics-api.md) to map metric name and namespace to an external object like an AWS SQS Queue. +There are a couple drawbacks to using the HPA’s API directly. The most obvious is the name, which would be more aptly called HorizontalAutoscaler. Most of its abstractions extend cleanly to Node Groups (e.g. [ScaleTargetRef](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [MetricTarget](https://godoc.org/k8s.io/api/autoscaling/v2beta2#MetricTarget), [ScalingPolicy](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingPolicy), [MinReplicas](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [MaxReplicas](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec), [Behavior](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerBehavior), [StabilizationWindowSeconds](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingRules)). Others require slight adjustments (e.g. [ScalingPolicyType](https://godoc.org/k8s.io/api/autoscaling/v2beta2#HPAScalingPolicyType) needs to be tweaked to refer to “replicas” instead of “pods”). However, [MetricSpec](https://godoc.org/k8s.io/api/autoscaling/v2beta2#MetricSpec) is specific to pods and requires changes if relied upon. MetricsSpec has four subfields corresponding to different metrics sources. [ResourceMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ResourceMetricSource), which uses the [Resource Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/resource-metrics-api.md) and provides CPU and memory for pods and nodes. [PodsMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#PodsMetricSource), which is syntactic sugar for [ObjectMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ObjectMetricSource), each of which each retrieve metrics from the [Custom Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/custom-metrics-api.md). [ExternalMetricSource](https://godoc.org/k8s.io/api/autoscaling/v2beta2#ExternalMetricSource), which uses the [External Metrics API](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/external-metrics-api.md) to map metric name and namespace to an external object like an AWS SQS Queue. One approach would be to use the MetricsSpec and its four sources as-is. This requires sourcing all metrics from the Kubernetes metrics APIs (see limitations above). It’s also somewhat awkward, as users would likely never use the PodsMetricSpec or ResourceMetricsSpec to scale their node groups. The primary reason to go this route is alignment with the HorizontalPodAutoscaler and existing Kubernetes metrics APIs. The current Kubernetes metrics architecture is arguably too pod specific and could be changed to be more generic, but we consider engagement with SIG Instrumentation to be out of scope for the short term. Another option would be use ObjectMetricsSpec and ExternalMetricsSpec and omit pod-specific metrics APIs. This generically covers metrics for both in-cluster and external objects (i.e. custom.metrics.k8s.io and external.metrics.k8s.io). This approach is cleaner from the perspective of a node autoscaler, but makes future alignment with the HPA more challenging. Pod metrics could still specified, but this removes the syntactic sugar that simplifies the most common use cases for pod autoscaling. -If we choose to integrate with directly with Prometheus metrics (discussed above), there will need to be a new option in the MetricsSpec to specify it as a metrics source (e.g PrometheusMetricSource). Customers would specify a [promql query](https://prometheus.io/docs/prometheus/latest/querying/basics/) to retrieve the metric. The decision to create a PrometheusMetricSource is orthogonal from whether or not we keep existing HPA metrics sources. Either way requires changes to the MetricsSpec; Prometheus support can be built alongside or replace existing metrics sources. +If we choose to integrate directly with Prometheus metrics (discussed above), there will need to be a new option in the MetricsSpec to specify it as a metrics source (e.g PrometheusMetricSource). Users would specify a [promql query](https://prometheus.io/docs/prometheus/latest/querying/basics/) to retrieve the metric. The decision to create a PrometheusMetricSource is orthogonal from whether or not we keep existing HPA metrics sources. Either way requires changes to the MetricsSpec; Prometheus support can be built alongside or replace existing metrics sources. We could also completely diverge from the HPA and start with a minimal autoscaler definition that covers initial node autoscaling use cases. This avoids premature abstraction of a generic autoscaling definition. However, we’re cautious to start from scratch, as it presumes we can design autoscaling APIs better than the HPA. It also makes alignment more challenging in the future. @@ -239,7 +239,7 @@ A more advanced algorithm called [Proportional Integral Derivative](https://en.w Predictive autoscaling is an experimental field that leverages machine learning to make scale decisions. This approach is used at [Netflix](https://netflixtechblog.com/scryer-netflixs-predictive-auto-scaling-engine-part-2-bb9c4f9b9385) to learn periodic traffic patterns by analyzing metrics like request per second. Theoretically, deep learning could be used in combination with a rich cluster metrics dataset (e.g. Prometheus) to produce high accuracy black box scale decisions. -The question of which algorithm to use is difficult to answer without deep research and experimentation with real customer workloads. Rather than staking a claim on any particular algorithm, we will leave the door open to iterate and improve options and the default for Karpenter’s autoscaling algorithm. We will initially implement a proportional algorithm. +The question of which algorithm to use is difficult to answer without deep research and experimentation with real user workloads. Rather than staking a claim on any particular algorithm, we will leave the door open to iterate and improve options and the default for Karpenter’s autoscaling algorithm. We will initially implement a proportional algorithm. ## APIs @@ -297,9 +297,9 @@ spec: ### Scalable Node Group -The decision to use the HPA’s scaleTargetRef concept creates two requirements for this resource. The API must represent a node group and must implement the [scale subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#subresources). The only responsibility of this resource is to control the replicas field, but this doesn’t preclude targeting a resource that is a full representation of the node group. There currently isn’t a Kubernetes native resource for node group, but there are a number of kubernetes ecosystem projects that model that node groups including [Kops Instance Groups](https://kops.sigs.k8s.io/tutorial/working-with-instancegroups/), [Cluster API Machine Pool](https://github.com/kubernetes-sigs/cluster-api/blob/master/docs/proposals/20190919-machinepool-api.md), [Amazon Controllers for Kubernetes (asg implementation tbd)](https://aws.amazon.com/blogs/containers/aws-controllers-for-kubernetes-ack/), [Anthos Config Management (node pool implementation tbd)](https://cloud.google.com/anthos-config-management/docs/overview), and others. +The decision to use the HPA’s scaleTargetRef concept creates two requirements for this resource. The API must represent a node group and must implement the [scale subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#subresources). The only responsibility of this resource is to control the replicas field, but this doesn’t preclude targeting a resource that is a full representation of the node group. There currently isn’t a Kubernetes native resource for node group, but there are a number of Kubernetes ecosystem projects that model node groups including [Kops Instance Groups](https://kops.sigs.k8s.io/tutorial/working-with-instancegroups/), [Cluster API Machine Pool](https://github.com/kubernetes-sigs/cluster-api/blob/master/docs/proposals/20190919-machinepool-api.md), [Amazon Controllers for Kubernetes (asg implementation tbd)](https://aws.amazon.com/blogs/containers/aws-controllers-for-kubernetes-ack/), [Anthos Config Management (node pool implementation tbd)](https://cloud.google.com/anthos-config-management/docs/overview), and others. -The Horizontal Autoscaler API is flexible to any and all of these options. However, many Kubernetes customers don’t rely on one of these mechanisms for node group management, instead relying on cloud provider specific abstractions (e.g. ASG). Therefore, we will introduce a resource that can be optionally targeted by HorizontalAutoscaler’s scaleTargetRef for users who don’t have a KRM node group concept. This is a stop gap until other node group representations become more widespread. This object will follow a cloud provider model to provide implementations for different cloud provider solutions such as EKS Managed Node Groups, EC2 Auto Scaling Groups, and others. The only field supported by this resource is replicas, leaving management responsibilities like upgrade up to some other controller. +The Horizontal Autoscaler API is flexible to any and all of these options. However, many Kubernetes users don’t rely on one of these mechanisms for node group management, instead relying on cloud provider specific abstractions (e.g. ASG). Therefore, we will introduce a resource that can be optionally targeted by HorizontalAutoscaler’s scaleTargetRef for users who don’t have a KRM node group concept. This is a stop gap until other node group representations become more widespread. This object will follow a cloud provider model to provide implementations for different cloud provider solutions such as EKS Managed Node Groups, EC2 Auto Scaling Groups, and others. The only field supported by this resource is replicas, leaving management responsibilities like upgrade up to some other controller. ``` apiVersion: karpenter.sh/v1alpha1 @@ -321,7 +321,7 @@ Capacity reservation is perhaps the most straightforward approach to node autosc Karpenter can automatically output capacity reservation metrics as they’re cheap to compute. This creates a zero-config starting point for users. As user requirements become more complex, capacity reservations can be used in conjunction with other signals. For example, capacity reservations can be used to drive scale down, while scale up is driven by pending pods. This mimics the Kubernetes Cluster Autoscaler’s algorithm. -Customers will be able to configure this as follows: +Users will be able to configure this as follows: ### Percentage overprovisioning @@ -391,7 +391,7 @@ The first is to drive pod autoscaling with a queue metric and node autoscaling w The second approach is to drive both pods and nodes using the same queue metric. This can be configured on a 1:1 basis, or as a ratio of n pods per node. The benefit of this approach over pending pods is in both latency and scalability. Pods and nodes are actively scaled up and the decision to scale up a node group is vastly simplified — it’s explicit, rather than implicit. -Queue-based node autoscaling and pending pods are both viable approaches with different trade offs. Pending pods is aligned with kubernetes bin-packing principles, and yields increased capacity utilization for clusters that host diverse workloads and are overprovisioned. In this case, new pods will first schedule into existing capacity before forcing node group scale up via a pending pod autoscaler. However, if users are looking for a simple batch processing workflow of scaleup → do work → scale down, the bin-packing benefits must be weighed against pending pods’ complexity, scalability, and scheduling latency tradeoffs. +Queue-based node autoscaling and pending pods are both viable approaches with different trade offs. Pending pods is aligned with Kubernetes bin-packing principles, and yields increased capacity utilization for clusters that host diverse workloads and are overprovisioned. In this case, new pods will first schedule into existing capacity before forcing node group scale up via a pending pod autoscaler. However, if users are looking for a simple batch processing workflow of scaleup → do work → scale down, the bin-packing benefits must be weighed against pending pods’ complexity, scalability, and scheduling latency tradeoffs. ``` apiVersion: karpenter.sh/v1alpha1 @@ -414,9 +414,9 @@ prometheus: ### Preemptable Nodes -Metrics driven autoscaling supports preemptable node architectures like AWS Spot. It’s incorrect to state that all metrics producers work flawlessly with preemptable nodes, but Karpenter’s flexibility gives customers the ability to apply preemption optimized metrics to preemptable node groups. +Metrics driven autoscaling supports preemptable node architectures like AWS Spot. It’s incorrect to state that all metrics producers work flawlessly with preemptable nodes, but Karpenter’s flexibility gives users the ability to apply preemption optimized metrics to preemptable node groups. -Preemptable nodes introduce new requirements; capacity is unavailable more frequently, and it can be be reclaimed by the cloud provider at any time. In autoscaling terms, this results in two cases: failure to scale up and forced scale down. A common solution to mitigate these problems is to rely on multiple preemptable instance types; if one becomes unavailable or removed, the autoscaler can scale up a new instance type that is available. Autoscaling algorithms require that instance types in the same node group are of the same shape (https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/#configuring-your-node-groups) (CPU, memory, etc). This limits the number of instance types that can be used in any given group, increasing the likelihood of insufficient capacity errors. Customers combat this by creating multiple preemptable node groups, each of a different shape. +Preemptable nodes introduce new requirements; capacity is unavailable more frequently, and it can be be reclaimed by the cloud provider at any time. In autoscaling terms, this results in two cases: failure to scale up and forced scale down. A common solution to mitigate these problems is to rely on multiple preemptable instance types; if one becomes unavailable or removed, the autoscaler can scale up a new instance type that is available. Autoscaling algorithms require that instance types in the same node group are of the same shape (https://aws.github.io/aws-eks-best-practices/cluster-autoscaling/#configuring-your-node-groups) (CPU, memory, etc). This limits the number of instance types that can be used in any given group, increasing the likelihood of insufficient capacity errors. Users combat this by creating multiple preemptable node groups, each of a different shape. One way to coordinate scaling across multiple node groups is to let the scheduler drive pod placement and use a capacity reservation metric for each node group. The horizontal autoscalers for each node group are not aware of each other, but they are aware of the pods that the scheduler has assigned to their nodes. As the scheduler adds pods to nodes, the corresponding node group will expand to maintain its reservation. If capacity for any node group becomes unavailable, the node group will fill up until the scheduler is forced to schedule elsewhere. This will gracefully fall back to node groups that have capacity and will continue to scale based off of their reservation metrics. @@ -470,11 +470,11 @@ Namespacing the HorizontalAutoscaler is nuanced. If we aspire to a global autosc The Node resource is not namespaced, so it might make sense to do the same for ScalableNodeGroup. Multiple ScalableNodeGroups pointing to the same cloud provider node group will result in undesired behavior. This could still happen if multiple conflicting resources were applied to the same namespace, but this scenario is much less likely. Given that MetricsProducer and HorizontalAutoscaler are both namespaced, it will provide a more intuitive user experience to namespace all three resources. -### Q: Should customers be able to apply multiple HorizontalAutoscaler configurations to the same scaleTargetRef? +### Q: Should users be able to apply multiple HorizontalAutoscaler configurations to the same scaleTargetRef? The Horizontal Autoscaler API has a []metrics field that lets users pass in multiple metrics. This allows users to specify an OR semantic to scale off of multiple signals. What about multiple Horizontal Autoscaler resources pointing to the same scaleTargetRef? For the HorizontalPodAutoscaler, this results in undesired behavior. For HorizontalAutoscaler, it’s possible to extend the OR semantic across multiple resources. The benefit would be that multiple application developers sharing a node group could scale the node group off of separate policies without being aware of each other. -It’s not clear whether or not this is an intuitive user experience. It’s arguable that this will lead to more confusion and questions of “why did my node group scale unexpectedly?”. We will await customer requests for this feature before considering it further. +It’s not clear whether or not this is an intuitive user experience. It’s arguable that this will lead to more confusion and questions of “why did my node group scale unexpectedly?”. We will await user requests for this feature before considering it further. ### Q: How can we make sure that Karpenter is horizontally scalable? diff --git a/docs/designs/scheduled_capacity.md b/docs/designs/scheduled_capacity.md new file mode 100644 index 000000000000..7105b12526ba --- /dev/null +++ b/docs/designs/scheduled_capacity.md @@ -0,0 +1,335 @@ +# Scheduled Capacity Design +## Introduction +Today, some Kubernetes users handle their workloads by scaling up and down in a recurring pattern. These patterns are +often indicative of some change in operational load and can come in the form of anything from a series of complex +scaling decisions to a one-off scale decision. + +## User Stories +* As a user I can periodically scale up and scale down my resources +* As a user I can schedule a special one-off scale request for my resources +* As a user I can utilize this metric in combination with others to schedule complex scaling decisions +* As a user I can see the current and future recommended states of my resources + +## Background +The important parts of Karpenter to take note of will be the HorizontalAutoscaler and the MetricsProducer. For any +user-specified resource, the MetricsProducer will be responsible for parsing the user input, calculating the metric +recommendation, and exposing it to the metrics endpoint. The HorizontalAutoscaler will be responsible for sending the +signals to scale the resource by using a `promql` query to grab the metric that the MetricsProducer has created. + +The core of each MetricsProducer is a reconcile loop, which runs at a pre-configured interval of time, and a record +function. The reconciliation ensures the metric is always being calculated, while the record function makes the data +available to the Prometheus server at every iteration of the loop. + +![](../images/scheduled-capacity-dataflow-diagram.png) + +While a HorizontalAutoscaler can only scale one resource, the metric that a MetricsProducer makes available can be used +by any amount of HorizontalAutoscalers. In addition, with a more complex `promql` +[query](https://prometheus.io/docs/prometheus/latest/querying/basics/), a user can also use a HorizontalAutoscaler to +scale based off multiple MetricsProducers. + +For more details, refer to [Karpenter’s design doc](./DESIGN.md). + +## Design +This design encompasses the `ScheduleSpec` and `ScheduledCapacityStatus` structs. The spec corresponds to the user +input specifying the scheduled behaviors. The status will be used as a way for the user to check the state of the +metric through `kubectl` commands. + +### Metrics Producer Spec +The `ScheduleSpec` is where the user will specify the times in which a schedule will activate and recommend what the +value of the metric should be. + +```{go} +type Timezone string + +type ScheduleSpec struct { + // Behaviors may be layered to achieve complex scheduling autoscaling logic + Behaviors []ScheduledBehavior `json:"behaviors"` + // Defaults to UTC. Users will specify their schedules assuming this is their timezone + // ref: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + // +optional + Timezone *Timezone `json:"timezone,omitempty"` + // A schedule defaults to this value when no behaviors are active + DefaultReplicas int32 `json:"defaultReplicas"` +} + +// ScheduledBehavior sets the metric to a replica value based on a start and end pattern. +type ScheduledBehavior struct { + // The value the MetricsProducer will emit when the current time is within start and end + Replicas int32 `json:"replicas"` + Start *Pattern `json:"start"` + End *Pattern `json:"end"` +} + +// Pattern is a strongly-typed version of crontabs +type Pattern struct { + // When minutes or hours are left out, they are assumed to match to 0 + Minutes *string `json:"minutes,omitempty"` + Hours *string `json:"hours,omitempty"` + // When Days, Months, or Weekdays are left out, + // they are represented by wildcards, meaning any time matches + Days *string `json:"days,omitempty"` + // List of 3-letter abbreviations i.e. Jan, Feb, Mar + Months *string `json:"months,omitempty"` + // List of 3-letter abbreviations i.e. "Mon, Tue, Wed" + Weekdays *string `json:"weekdays,omitempty"` +} +``` + +The spec below details how a user might configure their scheduled behaviors. The picture to the right corresponds to +the configuration. + +This configuration is scaling up for 9-5 on weekdays (red), scaling down a little at night (green), and then scaling +down almost fully for the weekends (blue). +![](../images/scheduled-capacity-example-schedule-graphic.png) +```{go} +apiVersion: autoscaling.karpenter.sh/v1alpha1 +kind: MetricsProducer +metadata: + name: scheduling +spec: + schedule: + timezone: America/Los_Angeles + defaultReplicas: 2 + behaviors: + // Scale way down on Friday evening for the weekend + - replicas: 1 + start: + weekdays: Fri + hours: 17 + end: + weekdays: Mon + hours: 9 + // Scale up on Weekdays for usual traffic + - replicas: 3 + start: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 9 + end: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 17 + // Scale down on weekday evenings but not as much as on weekends + - replicas: 2 + start: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 17 + end: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 9 +``` + +### Metrics Producer Status Struct +The `ScheduledCapacityStatus` can be used to monitor the MetricsProducer. The results of the algorithm will populate +this struct at every iteration of the reconcile loop. A user can see the values of this struct with +`kubectl get metricsproducers -oyaml`. +```{go} +type ScheduledCapacityStatus struct { + // The current recommendation - the metric the MetricsProducer is emitting + CurrentValue *int32 `json:"currentValue,omitempty"` + + // The time where CurrentValue will switch to NextValue + NextValueTime *apis.VolatileTime `json:"nextValueTime,omitempty"` + + // The next recommendation for the metric + NextValue *int32 `json:"nextValue,omitempty"` +} +``` + +## Algorithm Design +The algorithm will parse all behaviors and the start and end schedule formats. We find the `nextStartTime` and +`nextEndTime` for each of the schedules. These will be the times they next match in the future. + +We say a schedule matches if the following are all true: + +* The current time is before or equal to the `nextEndTime` +* The `nextStartTime` is after or equal to the `nextEndTime` + +Based on how many schedules match: + +* If there is no match, we set the metric to the `defaultReplicas` +* If there is only one match, we set the metric to that behavior’s value +* If there are multiple matches, we set the metric to the value that is specified first in the spec + +This algorithm and API choice are very similar to [KEDA’s Cron Scaler](https://keda.sh/docs/2.0/scalers/cron/). + +## Strongly-Typed vs Crontabs +Most other time-based schedulers use Crontabs as their API. This section discusses why we chose against Crontabs and +how the two choices are similar. + +* The [Cron library](https://github.com/robfig/cron) captures too broad of a scope for our use-case. + * In planning critical scaling decisions, freedom can hurt more than help. One malformed scale signal can cost the + user a lot more money, or even scale down unexpectedly. + * While our implementation will use the Cron library, picking a strongly-typed API will allows us to decide which + portions of the library we want to allow the users to configure. +* The wide range of functionality Cron provides is sometimes misunderstood +(e.g. [Crontab Pitfalls](##Crontab Pitfalls)). + * Adopting Crontab syntax adopts its pitfalls, which can be hard to fix in the future. + * If users have common problems involving Cron, it is more difficult to fix than if they were problems specific to + Karpenter. +* Karpenter’s metrics signals are best described as level-triggered. Crontabs were created to describe when to trigger +Cronjobs, which is best described as edge-triggered. + * If a user sees Crontabs, they may assume that Karpenter is edge-triggered behind the scenes, which + implies certain [problems](https://hackernoon.com/level-triggering-and-reconciliation-in-kubernetes-1f17fe30333d) + with availability. + * We want our users to infer correctly what is happening behind the scenes. + +## Field Plurality and Configuration Bloat +While Crontabs allow a user to specify **ranges** and **lists** of numbers/strings, we chose to **only** allow a **list** of +numbers/strings. Having a start and stop configuration in the form of Crontabs can confuse the user if they use overly +complex configurations. Reducing the scope of their choices to just a list of values can make it clearer. + +It is important to allow a user to specify multiple values to ease configuration load. While simpler cases like below +are easier to understand, adding more Crontab aspects like skip values and ranges can be much harder to mentally parse +at more complex levels of planning. We want to keep the tool intuitive, precise, and understandable, so that users who +understand their workloads can easily schedule them. + +```{go} +apiVersion: autoscaling.karpenter.sh/v1alpha1 +kind: MetricsProducer +metadata: + name: FakeScheduling +spec: + schedule: + timezone: America/Los_Angeles + defaultReplicas: 2 + behaviors: + // This spec WILL NOT work according to the design. + // Scale up on Weekdays for usual traffic + - replicas: 7 + start: + weekdays: Mon-Fri + hours: 9 + months: Jan-Mar + end: + weekdays: Mon-Fri + hours: 17 + months: Feb-Apr + // This spec WILL work according to the design. + // Scale down on weekday evenings + - replicas: 7 + start: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 9 + months: Jan,Feb,Mar + end: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: 17 + months: Feb,Mar,Apr +``` + +## FAQ +### How does this design handle collisions right now? + +* In the MVP, if a schedule ends when another starts, it will select the schedule that is starting. If more than one +are starting/valid, then it will use the schedule that comes first in the spec. +* Look at the Out of Scope https://quip-amazon.com/zQ7mAxg0wNDC/Karpenter-Periodic-Autoscaling#ANY9CAbqSLH below for +more details. + +### How would a priority system work for collisions? + +* Essentially, every schedule would have some associated Priority. If multiple schedules match to the same time, the +one with the higher priority will win. In the event of a tie, we resort to position in the spec. Whichever schedule is +configured first will win. + +### How can I leverage this tool to work with other metrics? + +* Using this metric in tandem with others is a part of the Karpenter HorizontalAutoscaler. There are many possibilities, + and it’s possible to do so with all metrics in prometheus, as long as they return an instant vector (a singular value). +* Let’s say a user is scaling based-off a queue (a metric currently supported by Karpenter). If they’d like to keep a +healthy minimum value regardless of the size of the queue to stay ready for an abnormally large batch of jobs, they can +configure their HorizontalAutoscaler’s Spec.Metrics.Prometheus.Query field to be the line below. + +`max(karpenter_queue_length{name="ml-training-queue"},karpenter_scheduled_capacity{name="schedules"})` + +### Is it required to use Prometheus? + +* Currently, Karpenter’s design has a dependency on Prometheus. We use Prometheus to store the data that the core design +components (MetricsProducer, HorizontalAutoscaler, ScalableNodeGroup) use to communicate with each other. + +### Why Karpenter HorizontalAutoscaler and MetricsProducer? Why not use the HPA? + +* Karpenter’s design details why we have a CRD called HorizontalAutoscaler, and how our MetricsProducers complement +them. While there are a lot of similarities, there are key differences as detailed in the design +[here](../designs/DESIGN.md#alignment-with-the-horizontal-pod-autoscaler-api). + +## Out of Scope - Additional Future Features +Our current design currently does not have a robust way to handle collisions and help visualize how the metric will look +over time. While these are important issues, their implementations will not be included in the MVP. + +### Collisions +Collisions occur when more than one schedule matches to the current time. When this happens, the MetricsProducer cannot +emit more than one value at a time, so it must choose one value. + +* When could collisions happen past user error? + * When a user wants a special one-off scale up request + * e.g. MyCompany normally has `x` replicas on all Fridays at 06:00 and `y` replicas on Fridays at 20:00, but + wants `z` replicas on Black Friday at 06:00 + * For only this Friday, the normal Friday schedule and this special one-off request will conflict +* Solutions for collision handling + * Create a warning with the first times that a collision could happen + * Doesn’t decide functionality for users + * Does not guarantee it will be resolved + * Associate each schedule with a priority which will be used in comparison to other colliding schedules + * Requires users to rank each of their schedules, which they may want to change based on the time they collide + * Ties in priority + * Use the order in which they’re specified in the spec **OR** + * Default to the defaultReplicas + +The only change to the structs from the initial design would be to add a Priority field in the ScheduledBehavior struct +as below. +```{go} +type ScheduledBehavior struct { + Replicas int32 `json:"replicas"` + Start *Pattern `json:"start"` + End *Pattern `json:"end"` + Priority *int32 `json:"priority,omitempty"` +} +``` + +### Configuration Complexity +When a user is configuring their resources, it’s easy to lose track of how the metric will look over time, especially +if a user may want to plan far into the future with many complex behaviors. Creating a tool to visualize schedules will +not only help users understand how their schedules will match up, but can ease concerns during the configuration +process. This can empower users to create even more complex schedules to match their needs. + +Possible Designs: + +* A dual standalone tool/UI that users can use to either validate or create their YAML + * Pros + * Allows check-as-you-go for configuration purposes + * Auto creates their YAML with a recommendation based + * Cons + * Requires users to manually use it + * Requires a lot more work to implement a UI to help create the YAML as opposed to just a tool to validate +* An extra function to be included as part of the MetricsProducerStatus + * Pros + * Always available to see the visualization with kubectl commands + * Cons + * Will use some compute power to keep running (may be trivial amount of compute) + * Cannot use to check-as-you-go for configuration purposes + +## Crontab Pitfalls +This design includes the choice to use a strongly-typed API due to cases where Crontabs do not act as expected. Below +is the most common misunderstanding. + +* Let's say I have a schedule to trigger on the following dates: + * Schedule A: First 3 Thursdays of January + * Schedule B: The Friday of the last week of January and the first 2 weeks of February + * Schedule C: Tuesday for every week until the end of March after Schedule B +* This is how someone might do it + * Schedule A - "* * 1-21 1 4" for the first three Thursdays of January + * Schedule B - "* * 22-31 1 5" for the last week of January and "* * 1-14 2 5" for the first two weeks of February + * Schedule C - "* * 15-31 2 2" for the last Tuesdays in February and "* * * 3 2" for the Tuesdays in March +* Problems with the above approach + * Schedule A will match to any day in January that is in 1/1 to 1/21 or is a Thursday + * Schedule B’s first crontab will match to any day in January that is in 1/22 to 1/31 or is a Friday + * Schedule B’s second crontab will match to any day in February that is in 2/1 to 2/14 or is a Friday + * Schedule C’s first crontab will match to any day in February that is in 2/15 to 2/31 or is a Tuesday + * Schedule C’s second crontab is the only one that works as intended. +* The way that crontabs are implemented is if both Dom and Dow are non-wildcards (as they are above in each of the +crontabs except for Schedule C’s second crontab), then the crontab is treated as a match if **either** the Dom **or** Dow +matches. + + + + + diff --git a/docs/examples/provisioner/provisioner.yaml b/docs/examples/provisioner/provisioner.yaml new file mode 100644 index 000000000000..f3ed9c0af0bf --- /dev/null +++ b/docs/examples/provisioner/provisioner.yaml @@ -0,0 +1,9 @@ +apiVersion: provisioning.karpenter.sh/v1alpha1 +kind: Provisioner +metadata: + name: example +spec: + cluster: + name: etarn-dev + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1ERXlNVEl3TURBMU1sb1hEVE14TURFeE9USXdNREExTWxvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTjVFCjJlRDhYTk82UmgrR053dEFMUFV4VEVxdnJ6SXRnNjZ2V0t4Q3JLQU43ZUwxTytpQmFCNUhNQmFKQzJZaHhkSzEKMXFha3RqMklRTUJ1QlBtdUNZV2E2VFRrN0ozSEFkeG9RcnhhVG1hVlAybytiZWwwR2piVGJQdCtvRUlWS2RJRApDcXc1NTBxR2pDU1FJa0ZHRXd4Y2o5SjRPUitzcFFDV2hGVmNQMnBjVkM3T1BKZkcwS0NzUTRGL1NPcW9ldWlmCkFDaWppWk04UG8zTEQrUThZNE9tZlJrdTRJR2xUSjUxNkdsSnZqaWJpd2JLcXMxY0hxLzYxVmR6TEdDbGY2SVIKM2wrNnN1WXpvYnFtdVg5emhnT0Jqa2M4NFJyWlpEQjQ2aGhnbTFmbHJ3V0diaFFNS2ozZW1kVmF5a2lRWDhyVApUTnI5c0R3SkRVRnU5R1RNd2hNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFMZkVzWUVKOS85QmJBcmdNWXBoWEtMV2prM3QKRmZ2MG8rbXU2M01maVg0Nm53dXdZYjBHQU8yRk9heDQyOVFrMDZpS2dnV0xJZmhKaGF0Vjhtb2hTTVFORzBLVQpBSWtwb3Y4emZXc0hLUjZ4dXZnQ2RhRE5zSFZkQlBZR3psUE9SamJvcUdJSHZLT3NGUEVJQW5lRHprSnUrL1pvCkpCcDVhaVVPREgremNvSy9YNkNxdXVjYXoydEFETHlMQlVJSEZMREZqZXB2SFVtSzVpbnppdDQ5dDJxUlpTR0QKcEVGeUsydWtxdklwbmtlazFCVzZlb2wzVmNVT29XRkZlS29pYnBjbHpZSElSYnJiQVR5MWdySHFRMjJIOGloVworaEsvQU5zNk9yeG9pVlBuZnlEcXhXUmw3ejhielhmZ29PczlWNGMrTkt1d2xmRW1DNVllRmV5bEgzaz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + endpoint: https://7D484C5616642C159A582437DDB71C28.gr7.us-west-2.eks.amazonaws.com diff --git a/docs/examples/queue-length-average-value.yaml b/docs/examples/queue-length-average-value.yaml index 5e62a97c29b0..5fc9014f8814 100644 --- a/docs/examples/queue-length-average-value.yaml +++ b/docs/examples/queue-length-average-value.yaml @@ -41,5 +41,8 @@ kind: ScalableNodeGroup metadata: name: ml-training-capacity spec: + # replicas is needed here + # +ref https://github.com/awslabs/karpenter/issues/64 + replicas: 1 type: AWSEKSNodeGroup id: arn:aws:eks:us-west-2:1234567890:nodegroup/test-cluster/training-capacity/qwertyuiop diff --git a/docs/examples/reserved-capacity-utilization.yaml b/docs/examples/reserved-capacity-utilization.yaml index f3c861e5fb45..219196d7e86c 100644 --- a/docs/examples/reserved-capacity-utilization.yaml +++ b/docs/examples/reserved-capacity-utilization.yaml @@ -11,7 +11,6 @@ spec: eks.amazonaws.com/nodegroup: default --- ### -# Track the queue_length metric. # Maintain 60% utilization of CPU or Memory (whichever is greater) # Set the desired replicas on scaleTargetRef of the Node Group below ### @@ -43,5 +42,8 @@ kind: ScalableNodeGroup metadata: name: microservices spec: + # replicas is needed here + # +ref https://github.com/awslabs/karpenter/issues/64 + replicas: 1 type: AWSEKSNodeGroup id: arn:aws:eks:us-west-2:1234567890:nodegroup:test-cluster/microservices/qwertyuiop diff --git a/docs/examples/scheduled-capacity.yaml b/docs/examples/scheduled-capacity.yaml new file mode 100644 index 000000000000..daa05d4f1287 --- /dev/null +++ b/docs/examples/scheduled-capacity.yaml @@ -0,0 +1,67 @@ +apiVersion: autoscaling.karpenter.sh/v1alpha1 +kind: MetricsProducer +metadata: + name: scheduling-example +spec: + scheduleSpec: + # default timezone is UTC + timezone: America/Los_Angeles + defaultReplicas: 1 + behaviors: + # leaving hours/minutes empty == specifying 0 + # leaving every other field empty == * + # single value (non-replicas) int fields need quotations + # Scale way down for the weekend + - replicas: 2 + start: + weekdays: fri + hours: "17" + end: + weekdays: mon + hours: "9" + # Scale way up for higher traffic for weekdays during work hours + - replicas: 6 + start: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: "9" + end: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: "17" + # Scale a little down for lower traffic for weekday evenings, but not as much as on the weekends + - replicas: 4 + start: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: "17" + end: + weekdays: Mon,Tue,Wed,Thu,Fri + hours: "9" +--- +apiVersion: autoscaling.karpenter.sh/v1alpha1 +kind: HorizontalAutoscaler +metadata: + name: scheduled-autoscaler-example +spec: + scaleTargetRef: + apiVersion: autoscaling.karpenter.sh/v1alpha1 + kind: ScalableNodeGroup + name: scheduled-nodegroup-example + minReplicas: 1 + maxReplicas: 10 + metrics: + - prometheus: + # Make sure name is equal to the name of your metricsproducer + query: karpenter_scheduled_replicas_value{name="scheduling-example"} + target: + type: AverageValue + value: 1 +--- +apiVersion: autoscaling.karpenter.sh/v1alpha1 +kind: ScalableNodeGroup +metadata: + name: scheduled-nodegroup-example +spec: + # replicas is needed here + # +ref https://github.com/awslabs/karpenter/issues/64 + replicas: 1 + type: AWSEKSNodeGroup + id: arn:aws:eks:us-west-2:112358132134:nodegroup/fibonacci/demo/qwertyuiop diff --git a/docs/images/karpenter-banner.png b/docs/images/karpenter-banner.png new file mode 100644 index 000000000000..af83cfa0a10f Binary files /dev/null and b/docs/images/karpenter-banner.png differ diff --git a/docs/images/karpenter-repo-preview.png b/docs/images/karpenter-repo-preview.png new file mode 100644 index 000000000000..724b68df6598 Binary files /dev/null and b/docs/images/karpenter-repo-preview.png differ diff --git a/docs/images/scheduled-capacity-dataflow-diagram.png b/docs/images/scheduled-capacity-dataflow-diagram.png new file mode 100644 index 000000000000..5e65475cf550 Binary files /dev/null and b/docs/images/scheduled-capacity-dataflow-diagram.png differ diff --git a/docs/images/scheduled-capacity-example-schedule-graphic.png b/docs/images/scheduled-capacity-example-schedule-graphic.png new file mode 100644 index 000000000000..4e12ec0aecd4 Binary files /dev/null and b/docs/images/scheduled-capacity-example-schedule-graphic.png differ diff --git a/docs/working-group/README.md b/docs/working-group/README.md new file mode 100644 index 000000000000..ba119917ff30 --- /dev/null +++ b/docs/working-group/README.md @@ -0,0 +1,164 @@ +# Working Group +Karpenter's community is open to everyone. All invites are managed through our [Calendar](https://calendar.google.com/calendar/u/0?cid=N3FmZGVvZjVoZWJkZjZpMnJrMmplZzVqYmtAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ). Alternatively, you can use our [iCal Export](https://calendar.google.com/calendar/ical/7qfdeof5hebdf6i2rk2jeg5jbk%40group.calendar.google.com/public/basic.ics) to add the events to Outlook or other email providers. + + +# Notes +Please contribute to our meeting notes by opening a PR. + +## Template +1. Community Questions +2. Work Items +3. Demos + +# Meeting notes (02/04/21) + +## Attendees: +- Prateek Gogia +- Viji Sarathy +- Subhrangshu Kumar Sarkar +- Shreyas Srinivasan +- Nathan Taber +- Ellis Tarn +- Nick Tran +- Brandon Wagner +- Guy Templeton + +## Notes + +- [Ellis] Karpenter received a bunch of customer feedback around pending pods problems + - Identified CA challenges + - Zones, mixed instance policies, nodes in node groups are assumed identical + - Limitations with ASGs with lots of groups + - New approach, replace ASG with direct create node API +- [Brandon] Supportive of the idea + - If we can have ASG and create fleet implementation for comparison +- [Ellis] SNG, HA, MP in karpenter are not compatible with this approach +- [Ellis] Focus on reducing pending pods latency to start +- [Guy] Opportunity for this approach, this removes all the guesswork and inaccuracy of CAS, which is quite honestly a pain in the ass. +- [Viji] More basic question, Autoscaling both nodes and pods. Scaling based on a single metric isn't enough. Using a pending pods + - How Karpenter and CA differs with pending pods approach + - [Viji] 2-3 minutes to start a new node with CA + - [Guy] 3 minutes for the nodes to schedulable + - [Ellis] m5.large took about 63 seconds with ready pods + - Create fleet is more modern API call with some parameters + - [Nathan] + - CA is slow in case of large clusters + - We have a requirement for compute resources and need that to be fullfiled by a service. + - Pre-selected ASG and shape sizes to create the nodes + - Strip the middle layers that translate the requirements and just ask for what we need. + - In cases when ASGs are not well pre thought out, CA is limited with the options available, whereas, Karpenter can make these decisions about the shape to select +- [Nathan] ASG wasn't built with the Kubernetes use case and sometimes works well and sometimes doesn't +- [Ellis] Allocator/ De-allocator model, dual controllers constantly provisioning new nodes for more capacity and removing under utilized nodes. +- [Guy] Dedicated node groups, taint ASGs, CA scales those up ASGs, can karpenter do it? + - [Ellis] When a pod has label and tolerations, we can create specific nodes with those labels and taints +- [Guy] Spot instances - how will that work with this model? + - We have dedicated node groups for istio gateways, rest is all spot. +- [Guy] CA and de-scheduler don't work nice with each other +- [Ellis] CA has 2 steps of configurations- ASGs and pods +- [Guy] Nicer approach, worry is how flexible that approach is? Seems like a very Google like approach of doing things with auto-provisioner. + - [Ellis] - Configuring every single pod with a label is a lot of work, compared to having taints at capacity. +- [Ellis] Provisioning and scheduling decisions- + - CA emulates scheduling and now karpenter knows instanceID + - We create a node object in Kubernetes and immediately bind the pod to the node and when the node becomes ready, pod image gets pulled and runs. + - Kube-scheduler is bypassed in this approach + - Simulations effort is not used when actual bin-packing is done by Kube-scheduler + - [Guy] Interesting approach, definetly sold on pod topology routing, can see benefits with bin-packing compared to guessing. + - You might end up more closely packed compared to the scheduler + - [Ellis] Scoring doesn't matter anymore because we don't have a set of nodes to select from + - [Subhu] How does node ready failure will be handled? + - Controller has node IDs and constantly compares with cloud provider nodes + - [Ellis] Bin packing is very cloud provider specific +- [Ellis] Spot termination happens when you get an event, de-allocator can be checking for pricing and improve costs with Spot. +- [Guy] Kops based instances are checking for health and draining nodes. Rebalance Recommendations are already handled by an internal KOPS at Skyscanner + +### In scope for Karpenter +- Pid Controller +- Upgrades +- Handle EC2 Instance Failures + +# Meeting notes (01/19/21) + +## Attendees: +- Ellis Tarn +- Jacob Gabrielson +- Subhrangshu Kumar Sarkar +- Prateek Gogia +- Nick Tran +- Brandon Wagner +- Guy Templeton + +## Notes: +- [Ellis] Conversation with Joe Burnett from sig-autoscaling + - HPA should work with scalable node group, as long you use an external metrics. + - POC is possible working with HPA +- [Ellis] Nick has made good progress in terms of API for scheduled scaling. + - Design review in upcoming weeks with the community. +- Change the meeting time to Thursday @9AM PT biweekly. + +# Meeting Notes (01/12/2021) + +## Attendees: +- Ellis Tarn +- Jacob Gabrielson +- Subhrangshu Kumar Sarkar +- Prateek Gogia +- Micah Hausler +- Viji Sarathy +- Shreyas Srinivasan +- Jeremy Cowan +- Guy Templeton + +## Discussions +- [Ellis] What are some common use cases for horizontal autoscaling like node auto-scaling? + - We have 2 metrics producer so far, SQS queue and utilization. + - Two are in pipeline, cron scheduling and pending pod metrics producers. +- [Jeremy] Have we looked at predictive scaling, analysing metrics overtime and scaling based on history? + - [Ellis] We are a little far from that, no work started on that yet +- [Viji] How can we pull cloudwatch metrics to Karpenter? + - [Ellis] We could have a cloud provider model to start with, to add cloudwatch support in horizontal autoscaler + - Other way would be external metrics API, you get one per cluster, creates problems within the ecosystem. + - [Viji] CP model pulls the metrics from the cloudwatch APIs and puts in the autoscaler? + - [Ellis] User would add info in the karpenter spec and an AWS client will try to load the metrics. + - External metrics API is easy, user has to figure how to configure with cloudwatch API. + - Universal metrics adapter supporting all the providers and prometheus. +- [Guy] Reg. external metrics API, there is a [proposal](https://github.com/kubernetes-sigs/custom-metrics-apiserver/issues/70) open in the community + - Custom cloud provider over gRPC [proposal](https://github.com/kubernetes/autoscaler/pull/3140) +- [Guy] Kops did something similar to what Ellis proposed. +- [Subhu] Are we going to support Pod Disruption Budget(PDB) or managed node groups (MNG) equivalent with other providers? + - [Ellis] karpenter will increase/decrease the number of nodes, someone needs to know which nodes to remove respecting the PDB. + - CA knows which nodes to scaled down it uses PDB. + - Node group is the right component deciding which node will not be violating PDB. +- [Guy] Other providers are rellying on PDB in CA for this support. It will be to good discuss with cluster API. +- [Ellis] We might have to support PDB if other providers don't support PDB in node group controllers to maintain neutrality. +- [Viji] Will try to get Karpenter installed and will look into cloudwatch integration. +- [Ellis] Looking to get feedback for installing Karpenter [demo](https://github.com/ellistarn/karpenter-aws-demo) +- [Ellis] Separate sync to discuss pending pods approach in Karpenter + - [Guy] Space for something less complex as compared to CA, there has been an explosion of flags in CA. + +# Meeting Notes (12/4/2020) + +## Attendees +@ellistarn +@prateekgogia +@gjtempleton +@shreyas87 + +## Notes: +- [Ellis] Shared background +- [Guy] Cloudwatch metrics, ECS scaling using cloudwatch metrics for autoscaling. +- [Guy] Karpenter supporting generic cloudwatch metrics? +- [Guy] Node autoscaling is supported? +- [Ellis] Cloud provider like model for cloudwatch, provider model exists in scalable node group side. +- [Ellis] Cloudwatch could support Prometheus API? +- [Ellis] We can have a direct cloudwatch integration and later refine it? +- [Guy] Implementing a generic cloud provider in core in CA. +- [Ellis] Will explore integration with cloudwatch directly, prefered will be coud provider model. +- [Guy] Contributions- People in squad will be interested, open to contribute features if it provides value to the team. +- [Guy] Scaling on non-pending pods and other resources, people have been asking. Karpenter looks promising for these aspects. +- [Ellis] - Long term goal, upstream project as an alternative. As open as possible and vendor neutral. +- [Guy] - There is a space for an alternative, given the history CA works around pending pods. Wider adoption possible if mature. +- [Ellis] - Landing point will be sig-autoscaling. +- [Guy] - CA lacks cron scheduling scaling. +- [Ellis] - pending pods are a big requirements. +- [Prateek] - introduced the pending pods producer proposal. +- [Ellis] - Move time earlier by an hour and change day to Thursday, create a GH issue to get feedback what time works? \ No newline at end of file diff --git a/go.mod b/go.mod index d088a626fbae..7a781190736c 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,13 @@ require ( github.com/go-logr/zapr v0.2.0 github.com/onsi/ginkgo v1.14.2 github.com/onsi/gomega v1.10.3 + github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_golang v1.8.0 github.com/prometheus/common v0.14.0 + github.com/robfig/cron/v3 v3.0.0 go.uber.org/multierr v1.6.0 go.uber.org/zap v1.16.0 + gopkg.in/retry.v1 v1.0.3 k8s.io/api v0.19.3 k8s.io/apimachinery v0.19.3 k8s.io/client-go v0.19.3 diff --git a/go.sum b/go.sum index fc079e0e2143..a3322dea4f01 100644 --- a/go.sum +++ b/go.sum @@ -69,6 +69,7 @@ github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.12 h1:qpxQ/DXfgsTNSYn8mUaCgQiJkCjBP8iHKw5ju+wkucU= github.com/aws/aws-sdk-go v1.35.12/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= +github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -80,6 +81,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -138,6 +140,8 @@ github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.2.2 h1:xfmOhhoH5fGPgbEAlhLpJH9p0z/0Qizio9osmvn9IUY= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -246,6 +250,7 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= @@ -445,6 +450,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -504,6 +511,10 @@ github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULU github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -513,33 +524,24 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= @@ -547,11 +549,9 @@ github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3 github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -568,7 +568,6 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738 h1:VcrIfasaLFkyjk6KNlXQSzO+B0fZcnECiDrKJsfxka0= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -577,13 +576,9 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -591,7 +586,6 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -599,7 +593,6 @@ go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= @@ -668,17 +661,13 @@ golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M= golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -687,7 +676,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -719,9 +707,7 @@ golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299 h1:DYfZAGf2WMFjMxbgTjaC+2HC7NkNAQs+6Q8b9WEB/F4= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -730,13 +716,11 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211 h1:9UQO31fZ+0aKQOFldThf7BKPM golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= @@ -789,9 +773,7 @@ google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= @@ -803,7 +785,6 @@ google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -817,7 +798,6 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -826,12 +806,10 @@ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -839,13 +817,14 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogR gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs= +gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -860,9 +839,7 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -871,18 +848,15 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.3 h1:GN6ntFnv44Vptj/b+OnMW7FmzkpDoIDLZRvKX3XH9aU= k8s.io/api v0.19.3/go.mod h1:VF+5FT1B74Pw3KxMdKyinLo+zynBaMBiAfGMuldcNDs= k8s.io/apiextensions-apiserver v0.19.2 h1:oG84UwiDsVDu7dlsGQs5GySmQHCzMhknfhFExJMz9tA= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apimachinery v0.19.2 h1:5Gy9vQpAGTKHPVOh5c4plE274X8D/6cuEiTO2zve7tc= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.3 h1:bpIQXlKjB4cB/oNpnNnV+BybGPR7iP5oYpsOTEJ4hgc= k8s.io/apimachinery v0.19.3/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= -k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= k8s.io/client-go v0.19.3 h1:ctqR1nQ52NUs6LpI0w+a5U+xjYwflFwA13OJKcicMxg= k8s.io/client-go v0.19.3/go.mod h1:+eEMktZM+MG0KO+PTkci8xnbCZHvj9TqR6Q1XDUIJOM= @@ -890,13 +864,11 @@ k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZ k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800 h1:9ZNvfPvVIEsp/T1ez4GQuzCcCTEQWhovSofhqR73A6g= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/hack/quick-install.sh b/hack/quick-install.sh index 286f23090a3b..2e35c7dda45f 100755 --- a/hack/quick-install.sh +++ b/hack/quick-install.sh @@ -1,23 +1,20 @@ #!/bin/bash set -eu -o pipefail -TEMP_DIR=$(mktemp -d) -trap "rm -rf $TEMP_DIR" EXIT - main() { local command=${1:-'--apply'} - if [[ $command = "--usage" ]]; then - usage - elif [[ $command = "--apply" ]]; then + if [[ $command = "--apply" ]]; then + echo "Installing Karpenter & dependencies.." apply echo "Installation complete!" elif [[ $command = "--delete" ]]; then + echo "Uninstalling Karpenter & dependencies.." delete echo "Uninstallation complete!" else echo "Error: invalid argument: $command" >&2 usage - exit 22 # EINVAL + exit 22 # EINVAL fi } @@ -25,7 +22,6 @@ usage() { cat <` apply() { helm repo add jetstack https://charts.jetstack.io helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo add karpenter https://awslabs.github.io/karpenter/charts helm repo update - certmanager - prometheus - make apply -} - -# If this fails you may have an old installation hanging around. If it's just for -# testing, you can remove it with something like this (match the version to the version -# you installed). -# -# VERSION=$(kubectl get deployment cert-manager -n cert-manager -ojsonpath='{.spec.template.spec.containers[0].image}{"\n"}' | cut -f2 -d:) -# kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/${VERSION}/cert-manager.yaml -certmanager() { helm upgrade --install cert-manager jetstack/cert-manager \ - --atomic \ --create-namespace \ --namespace cert-manager \ - --version v1.0.0 \ + --version v1.1.0 \ --set installCRDs=true -} -prometheus() { - # Minimal install of prometheus operator. helm upgrade --install kube-prometheus-stack prometheus-community/kube-prometheus-stack \ - --atomic \ --create-namespace \ --namespace monitoring \ --version 9.4.5 \ @@ -84,6 +68,9 @@ prometheus() { --set kubeStateMetrics.enabled=false \ --set nodeExporter.enabled=false \ --set prometheus.enabled=false + + helm upgrade --install karpenter karpenter/karpenter } +usage main "$@" diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index b5feb4582c18..6bfe6b0ba7de 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -17,6 +17,7 @@ package apis import ( "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + provisioningV1alpha1 "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" "k8s.io/apimachinery/pkg/runtime" ) @@ -30,4 +31,5 @@ func AddToScheme(s *runtime.Scheme) error { func init() { AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme) + AddToSchemes = append(AddToSchemes, provisioningV1alpha1.SchemeBuilder.AddToScheme) } diff --git a/pkg/apis/autoscaling/v1alpha1/horizontalautoscaler_status.go b/pkg/apis/autoscaling/v1alpha1/horizontalautoscaler_status.go index 461a9278f682..7cb1f2df5874 100644 --- a/pkg/apis/autoscaling/v1alpha1/horizontalautoscaler_status.go +++ b/pkg/apis/autoscaling/v1alpha1/horizontalautoscaler_status.go @@ -28,11 +28,11 @@ type HorizontalAutoscalerStatus struct { // CurrentReplicas is current number of replicas of pods managed by this // autoscaler, as last seen by the autoscaler. // +optional - CurrentReplicas int32 `json:"currentReplicas"` + CurrentReplicas *int32 `json:"currentReplicas"` // DesiredReplicas is the desired number of replicas of pods managed by this // autoscaler, as last calculated by the autoscaler. // +optional - DesiredReplicas int32 `json:"desiredReplicas"` + DesiredReplicas *int32 `json:"desiredReplicas"` // CurrentMetrics is the last read state of the metrics used by this // autoscaler. // +optional diff --git a/pkg/apis/autoscaling/v1alpha1/metricsproducer.go b/pkg/apis/autoscaling/v1alpha1/metricsproducer.go index 3ab3c7dda01b..a8b6aa4671f5 100644 --- a/pkg/apis/autoscaling/v1alpha1/metricsproducer.go +++ b/pkg/apis/autoscaling/v1alpha1/metricsproducer.go @@ -31,9 +31,9 @@ type MetricsProducerSpec struct { // to available resources for the nodes of a specified node group. // +optional ReservedCapacity *ReservedCapacitySpec `json:"reservedCapacity,omitempty"` - // ScheduledCapacity produces a metric according to a specified schedule. + // Schedule produces a metric according to a specified schedule. // +optional - ScheduledCapacity *ScheduledCapacitySpec `json:"scheduledCapacity,omitempty"` + Schedule *ScheduleSpec `json:"scheduleSpec,omitempty"` } type ReservedCapacitySpec struct { @@ -46,17 +46,36 @@ type PendingCapacitySpec struct { NodeSelector map[string]string `json:"nodeSelector"` } -type ScheduledCapacitySpec struct { - // NodeSelector specifies a node group. The selector must uniquely identify a set of nodes. - NodeSelector map[string]string `json:"nodeSelector"` +type ScheduleSpec struct { // Behaviors may be layered to achieve complex scheduling autoscaling logic Behaviors []ScheduledBehavior `json:"behaviors"` + // Defaults to UTC. Users will specify their schedules assuming this is their timezone + // ref: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones + // +optional + Timezone *string `json:"timezone,omitempty"` + // A schedule defaults to this value when no behaviors are active + DefaultReplicas int32 `json:"defaultReplicas"` } -// ScheduledBehavior defines a crontab which sets the metric to a specific replica value on a schedule. +// ScheduledBehavior sets the metric to a replica value based on a start and end pattern. type ScheduledBehavior struct { - Crontab string `json:"crontab"` - Replicas int32 `json:"replicas"` + // The value the MetricsProducer will emit when the current time is within start and end + Replicas int32 `json:"replicas"` + Start *Pattern `json:"start"` + End *Pattern `json:"end"` +} + +// Pattern is a strongly-typed version of crontabs +type Pattern struct { + // When minutes or hours are left out, they are assumed to match to 0 + Minutes *string `json:"minutes,omitempty"` + Hours *string `json:"hours,omitempty"` + // When Days, Months, or Weekdays are left out, they are represented by wildcards, meaning any time matches + Days *string `json:"days,omitempty"` + // List of 3-letter abbreviations i.e. Jan, Feb, Mar + Months *string `json:"months,omitempty"` + // List of 3-letter abbreviations i.e. "Mon, Tue, Wed" + Weekdays *string `json:"weekdays,omitempty"` } // PendingPodsSpec outputs a metric that identifies scheduling opportunities for pending pods in specified node groups. diff --git a/pkg/apis/autoscaling/v1alpha1/metricsproducer_defaults.go b/pkg/apis/autoscaling/v1alpha1/metricsproducer_defaults.go index 33a69ed018da..d3830fa67410 100644 --- a/pkg/apis/autoscaling/v1alpha1/metricsproducer_defaults.go +++ b/pkg/apis/autoscaling/v1alpha1/metricsproducer_defaults.go @@ -12,10 +12,54 @@ See the License for the specific language governing permissions and limitations under the License. */ -// +kubebuilder:webhook:path=/mutate-autoscaling-karpenter-sh-v1alpha1-metricsproducer,mutating=true,sideEffects=None,failurePolicy=fail,groups=autoscaling.karpenter.sh,resources=metricsproducers,verbs=create;update,versions=v1alpha1,name=mmetricsproducer.kb.io +// +kubebuilder:webhook:verbs=create;update,path=/mutate-autoscaling-karpenter-sh-v1alpha1-metricsproducer,mutating=true,sideEffects=None,failurePolicy=fail,groups=autoscaling.karpenter.sh,resources=metricsproducers,versions=v1alpha1,name=mmetricsproducer.kb.io package v1alpha1 -// Default implements webhook.Defaulter so a webhook will be registered for the type -func (r *MetricsProducer) Default() { +import ( + "knative.dev/pkg/ptr" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/webhook" +) + +var _ webhook.Defaulter = &MetricsProducer{} + +type specDefaulter interface { + defaultValues() +} + +// Default implements webhook.Defaulter so a webhook will be registered +func (m *MetricsProducer) Default() { + m.defaultValues() +} + +// Default PendingCapacity +func (s *PendingCapacitySpec) defaultValues() { +} + +// Default ReservedCapacity +func (s *ReservedCapacitySpec) defaultValues() { +} + +// Default ScheduleSpec +func (s *ScheduleSpec) defaultValues() { + if s.Timezone == nil { + s.Timezone = ptr.String("UTC") + } +} + +func (s *QueueSpec) defaultValues() { +} + +func (m *MetricsProducer) defaultValues() { + for _, defaulter := range []specDefaulter{ + m.Spec.PendingCapacity, + m.Spec.ReservedCapacity, + m.Spec.Schedule, + m.Spec.Queue, + } { + if !reflect.ValueOf(defaulter).IsNil() { + defaulter.defaultValues() + } + } } diff --git a/pkg/apis/autoscaling/v1alpha1/metricsproducer_status.go b/pkg/apis/autoscaling/v1alpha1/metricsproducer_status.go index 7bf249dbcc8b..63620ab05894 100644 --- a/pkg/apis/autoscaling/v1alpha1/metricsproducer_status.go +++ b/pkg/apis/autoscaling/v1alpha1/metricsproducer_status.go @@ -48,6 +48,16 @@ type QueueStatus struct { } type ScheduledCapacityStatus struct { + // The current recommendation - the metric the MetricsProducer is emitting + CurrentValue *int32 `json:"currentValue,omitempty"` + + // Not Currently Implemented + // The time in the future where CurrentValue will switch to NextValue + NextValueTime *apis.VolatileTime `json:"nextValueTime,omitempty"` + + // Not Currently Implemented + // The next recommendation for the metric + NextValue *int32 `json:"nextValue,omitempty"` } // We use knative's libraries for ConditionSets to manage status conditions. diff --git a/pkg/apis/autoscaling/v1alpha1/metricsproducer_validation.go b/pkg/apis/autoscaling/v1alpha1/metricsproducer_validation.go index f58309b5e149..981d4660485a 100644 --- a/pkg/apis/autoscaling/v1alpha1/metricsproducer_validation.go +++ b/pkg/apis/autoscaling/v1alpha1/metricsproducer_validation.go @@ -12,40 +12,71 @@ See the License for the specific language governing permissions and limitations under the License. */ +// +kubebuilder:webhook:verbs=create;update,path=/validate-autoscaling-karpenter-sh-v1alpha1-metricsproducer,mutating=false,sideEffects=None,failurePolicy=fail,groups=autoscaling.karpenter.sh,resources=metricsproducers,versions=v1alpha1,name=vmetricsproducer.kb.io package v1alpha1 -import "fmt" +import ( + "fmt" + "k8s.io/apimachinery/pkg/runtime" + "reflect" + "regexp" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "strings" + "time" +) -// +kubebuilder:object:generate=false -type QueueValidator func(*QueueSpec) error +var _ webhook.Validator = &MetricsProducer{} -var ( - queueValidator = map[QueueType]QueueValidator{} -) +type specValidator interface { + validate() error +} -func RegisterQueueValidator(queueType QueueType, validator QueueValidator) { - queueValidator[queueType] = validator +// Only Validates ScheduledCapacity MetricsProducer right now +func (m *MetricsProducer) ValidateCreate() error { + return m.validate() } -// Validate Queue -func (mp *MetricsProducerSpec) Validate() error { - if mp.Queue != nil { - queueValidate, ok := queueValidator[mp.Queue.Type] - if !ok { - return fmt.Errorf("unexpected queue type %v", mp.Queue.Type) - } - if err := queueValidate(mp.Queue); err != nil { - return fmt.Errorf("invalid Metrics Producer, %w", err) +func (m *MetricsProducer) ValidateUpdate(old runtime.Object) error { + return m.validate() +} + +func (m *MetricsProducer) ValidateDelete() error { + return nil +} + +func (m *MetricsProducer) validate() error { + for _, validator := range []specValidator{ + m.Spec.PendingCapacity, + m.Spec.ReservedCapacity, + m.Spec.Schedule, + } { + if !reflect.ValueOf(validator).IsNil() { + return validator.validate() } } - if mp.PendingCapacity != nil { - return mp.PendingCapacity.validate() + return nil +} + +// Validate ScheduleSpec +func (s *ScheduleSpec) validate() error { + for _, b := range s.Behaviors { + if err := b.Start.validate(); err != nil { + return fmt.Errorf("start pattern could not be parsed, %w", err) + } + if err := b.End.validate(); err != nil { + return fmt.Errorf("end pattern could not be parsed, %w", err) + } + if b.Replicas < 0 { + return fmt.Errorf("behavior.replicas cannot be negative") + } } - if mp.ReservedCapacity != nil { - return mp.ReservedCapacity.validate() + if s.DefaultReplicas < 0 { + return fmt.Errorf("defaultReplicas cannot be negative") } - if mp.ScheduledCapacity != nil { - return mp.ScheduledCapacity.validate() + if s.Timezone != nil { + if _, err := time.LoadLocation(*s.Timezone); err != nil { + return fmt.Errorf("timezone region could not be parsed") + } } return nil } @@ -63,7 +94,73 @@ func (s *ReservedCapacitySpec) validate() error { return nil } -// Validate ScheduledCapacity -func (s *ScheduledCapacitySpec) validate() error { +// These regex patterns are meant to match to one element at a time +const ( + weekdayRegexPattern = "^((sun(day)?|0|7)|(mon(day)?|1)|(tue(sday)?|2)|(wed(nesday)?|3)|(thu(rsday)?|4)|(fri(day)?|5)|(sat(urday)?|6))$" + monthRegexPattern = "^((jan(uary)?|1)|(feb(ruary)?|2)|(mar(ch)?|3)|(apr(il)?|4)|(may|5)|(june?|6)|(july?|7)|(aug(ust)?|8)|(sep(tember)?|9)|((oct(ober)?)|(10))|(nov(ember)?|(11))|(dec(ember)?|(12)))$" + onlyNumbersPattern = `^\d+$` +) + +var regexMap = map[string]string{ + "Weekdays": weekdayRegexPattern, + "Months": monthRegexPattern, + "Days": onlyNumbersPattern, + "Hours": onlyNumbersPattern, + "Minutes": onlyNumbersPattern, +} + +func (p *Pattern) validate() error { + val := reflect.ValueOf(p) + for _, name := range []string{"Weekdays", "Months", "Days", "Hours", "Minutes"} { + ptr := reflect.Indirect(val).FieldByName(name) + if ptr.IsNil() { + continue + } + field := ptr.Elem().String() + if !isValidField(&field, regexMap[name]) { + return fmt.Errorf("unable to parse: %s", field) + } + } + return nil +} + +func isValidField(field *string, regexPattern string) bool { + if field == nil { + return true + } + elements := strings.Split(*field, ",") + if len(elements) == 0 { + return false + } + for _, elem := range elements { + elem = strings.ToLower(strings.Trim(elem, " ")) + matched, _ := regexp.MatchString(regexPattern, elem) + if !matched { + return false + } + } + return true +} + +// +kubebuilder:object:generate=false +type QueueValidator func(*QueueSpec) error + +var ( + queueValidator = map[QueueType]QueueValidator{} +) + +func RegisterQueueValidator(queueType QueueType, validator QueueValidator) { + queueValidator[queueType] = validator +} + +// Validate at different level for cloud provider +func (mp *MetricsProducerSpec) ValidateQueue() error { + queueValidate, ok := queueValidator[mp.Queue.Type] + if !ok { + return fmt.Errorf("unexpected queue type %v", mp.Queue.Type) + } + if err := queueValidate(mp.Queue); err != nil { + return fmt.Errorf("invalid Metrics Producer, %w", err) + } return nil } diff --git a/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_status.go b/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_status.go index 417ef4a75265..44af704df8ab 100644 --- a/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_status.go +++ b/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_status.go @@ -22,7 +22,7 @@ type ScalableNodeGroupStatus struct { // Replicas displays the actual size of the ScalableNodeGroup // at the time of the last reconciliation // +optional - Replicas int32 `json:"replicas,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` // Conditions is the set of conditions required for the scalable node group // to successfully enforce the replica count of the underlying group // +optional diff --git a/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_validation.go b/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_validation.go index ccc29bed7756..928b33881a7a 100644 --- a/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_validation.go +++ b/pkg/apis/autoscaling/v1alpha1/scalablenodegroup_validation.go @@ -12,12 +12,29 @@ See the License for the specific language governing permissions and limitations under the License. */ +// +kubebuilder:webhook:verbs=create;update,path=/validate-autoscaling-karpenter-sh-v1alpha1-scalablenodegroup,mutating=false,sideEffects=None,failurePolicy=fail,groups=autoscaling.karpenter.sh,resources=scalablenodegroups,versions=v1alpha1,name=scalablenodegroup.kb.io package v1alpha1 import ( "fmt" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) +var _ webhook.Validator = &ScalableNodeGroup{} + +func (sng *ScalableNodeGroup) ValidateCreate() error { + return nil +} + +func (sng *ScalableNodeGroup) ValidateUpdate(old runtime.Object) error { + return nil +} + +func (sng *ScalableNodeGroup) ValidateDelete() error { + return nil +} + // +kubebuilder:object:generate=false type ScalableNodeGroupValidator func(*ScalableNodeGroupSpec) error diff --git a/pkg/apis/autoscaling/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/autoscaling/v1alpha1/zz_generated.deepcopy.go index 2b4778d1f6b4..8ccf4218b712 100644 --- a/pkg/apis/autoscaling/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/autoscaling/v1alpha1/zz_generated.deepcopy.go @@ -155,6 +155,16 @@ func (in *HorizontalAutoscalerStatus) DeepCopyInto(out *HorizontalAutoscalerStat *out = new(apis.VolatileTime) (*in).DeepCopyInto(*out) } + if in.CurrentReplicas != nil { + in, out := &in.CurrentReplicas, &out.CurrentReplicas + *out = new(int32) + **out = **in + } + if in.DesiredReplicas != nil { + in, out := &in.DesiredReplicas, &out.DesiredReplicas + *out = new(int32) + **out = **in + } if in.CurrentMetrics != nil { in, out := &in.CurrentMetrics, &out.CurrentMetrics *out = make([]MetricStatus, len(*in)) @@ -358,9 +368,9 @@ func (in *MetricsProducerSpec) DeepCopyInto(out *MetricsProducerSpec) { *out = new(ReservedCapacitySpec) (*in).DeepCopyInto(*out) } - if in.ScheduledCapacity != nil { - in, out := &in.ScheduledCapacity, &out.ScheduledCapacity - *out = new(ScheduledCapacitySpec) + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(ScheduleSpec) (*in).DeepCopyInto(*out) } } @@ -398,7 +408,7 @@ func (in *MetricsProducerStatus) DeepCopyInto(out *MetricsProducerStatus) { if in.ScheduledCapacity != nil { in, out := &in.ScheduledCapacity, &out.ScheduledCapacity *out = new(ScheduledCapacityStatus) - **out = **in + (*in).DeepCopyInto(*out) } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions @@ -419,6 +429,46 @@ func (in *MetricsProducerStatus) DeepCopy() *MetricsProducerStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Pattern) DeepCopyInto(out *Pattern) { + *out = *in + if in.Minutes != nil { + in, out := &in.Minutes, &out.Minutes + *out = new(string) + **out = **in + } + if in.Hours != nil { + in, out := &in.Hours, &out.Hours + *out = new(string) + **out = **in + } + if in.Days != nil { + in, out := &in.Days, &out.Days + *out = new(string) + **out = **in + } + if in.Months != nil { + in, out := &in.Months, &out.Months + *out = new(string) + **out = **in + } + if in.Weekdays != nil { + in, out := &in.Weekdays, &out.Weekdays + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Pattern. +func (in *Pattern) DeepCopy() *Pattern { + if in == nil { + return nil + } + out := new(Pattern) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PendingCapacitySpec) DeepCopyInto(out *PendingCapacitySpec) { *out = *in @@ -644,6 +694,11 @@ func (in *ScalableNodeGroupSpec) DeepCopy() *ScalableNodeGroupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScalableNodeGroupStatus) DeepCopyInto(out *ScalableNodeGroupStatus) { *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make(apis.Conditions, len(*in)) @@ -709,43 +764,53 @@ func (in *ScalingRules) DeepCopy() *ScalingRules { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ScheduledBehavior) DeepCopyInto(out *ScheduledBehavior) { +func (in *ScheduleSpec) DeepCopyInto(out *ScheduleSpec) { *out = *in + if in.Behaviors != nil { + in, out := &in.Behaviors, &out.Behaviors + *out = make([]ScheduledBehavior, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Timezone != nil { + in, out := &in.Timezone, &out.Timezone + *out = new(string) + **out = **in + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledBehavior. -func (in *ScheduledBehavior) DeepCopy() *ScheduledBehavior { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduleSpec. +func (in *ScheduleSpec) DeepCopy() *ScheduleSpec { if in == nil { return nil } - out := new(ScheduledBehavior) + out := new(ScheduleSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ScheduledCapacitySpec) DeepCopyInto(out *ScheduledCapacitySpec) { +func (in *ScheduledBehavior) DeepCopyInto(out *ScheduledBehavior) { *out = *in - if in.NodeSelector != nil { - in, out := &in.NodeSelector, &out.NodeSelector - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } + if in.Start != nil { + in, out := &in.Start, &out.Start + *out = new(Pattern) + (*in).DeepCopyInto(*out) } - if in.Behaviors != nil { - in, out := &in.Behaviors, &out.Behaviors - *out = make([]ScheduledBehavior, len(*in)) - copy(*out, *in) + if in.End != nil { + in, out := &in.End, &out.End + *out = new(Pattern) + (*in).DeepCopyInto(*out) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledCapacitySpec. -func (in *ScheduledCapacitySpec) DeepCopy() *ScheduledCapacitySpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledBehavior. +func (in *ScheduledBehavior) DeepCopy() *ScheduledBehavior { if in == nil { return nil } - out := new(ScheduledCapacitySpec) + out := new(ScheduledBehavior) in.DeepCopyInto(out) return out } @@ -753,6 +818,21 @@ func (in *ScheduledCapacitySpec) DeepCopy() *ScheduledCapacitySpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScheduledCapacityStatus) DeepCopyInto(out *ScheduledCapacityStatus) { *out = *in + if in.CurrentValue != nil { + in, out := &in.CurrentValue, &out.CurrentValue + *out = new(int32) + **out = **in + } + if in.NextValueTime != nil { + in, out := &in.NextValueTime, &out.NextValueTime + *out = new(apis.VolatileTime) + (*in).DeepCopyInto(*out) + } + if in.NextValue != nil { + in, out := &in.NextValue, &out.NextValue + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledCapacityStatus. diff --git a/pkg/apis/provisioning/v1alpha1/doc.go b/pkg/apis/provisioning/v1alpha1/doc.go new file mode 100644 index 000000000000..0c110d7fc5b4 --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/doc.go @@ -0,0 +1,56 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1alpha1 contains API Schema definitions for the v1alpha1 API group +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen=package,register +// +k8s:defaulter-gen=TypeMeta +// +groupName=provisioning.karpenter.sh +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "knative.dev/pkg/apis" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // APIVersion is the current API version used to register these objects + APIVersion = "v1alpha1" + + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "provisioning.karpenter.sh", Version: APIVersion} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} + + // AddToScheme is required by pkg/client/... + AddToScheme = SchemeBuilder.AddToScheme +) + +const ( + // Active is a condition implemented by all resources. It indicates that the + // controller is able to take actions: it's correctly configured, can make + // necessary API calls, and isn't disabled. + Active apis.ConditionType = "Active" +) + +// Resource is required by pkg/client/listers/... +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +func init() { + SchemeBuilder.Register(&Provisioner{}, &ProvisionerList{}) +} diff --git a/pkg/apis/provisioning/v1alpha1/provisioner.go b/pkg/apis/provisioning/v1alpha1/provisioner.go new file mode 100644 index 000000000000..e3dbf44a8d26 --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/provisioner.go @@ -0,0 +1,68 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ProvisionerSpec struct { + // +optional + Cluster *ClusterSpec `json:"cluster,omitempty"` + Allocator AllocatorSpec `json:"allocator,omitempty"` + Reallocator ReallocatorSpec `json:"reallocator,omitempty"` +} + +// ClusterSpec configures the cluster that the provisioner operates against. If +// not specified, it will default to using the controller's kube-config. +type ClusterSpec struct { + // Name is required to detect implementing cloud provider resources. + // +required + Name string `json:"name"` + // CABundle is required for nodes to verify API Server certificates. + // +required + CABundle string `json:"caBundle"` + // Endpoint is required for nodes to connect to the API Server. + // +required + Endpoint string `json:"endpoint"` +} + +// AllocatorSpec configures node allocation policy +type AllocatorSpec struct { + InstanceTypes []string `json:"instanceTypes,omitempty"` +} + +// ReallocatorSpec configures node reallocation policy +type ReallocatorSpec struct { +} + +// Provisioner is the Schema for the Provisioners API +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +type Provisioner struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ProvisionerSpec `json:"spec,omitempty"` + Status ProvisionerStatus `json:"status,omitempty"` +} + +// ProvisionerList contains a list of Provisioner +// +kubebuilder:object:root=true +type ProvisionerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Provisioner `json:"items"` +} diff --git a/pkg/apis/provisioning/v1alpha1/provisioner_defaults.go b/pkg/apis/provisioning/v1alpha1/provisioner_defaults.go new file mode 100644 index 000000000000..770d868cd731 --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/provisioner_defaults.go @@ -0,0 +1,18 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +func (r *Provisioner) Default() { +} diff --git a/pkg/apis/provisioning/v1alpha1/provisioner_status.go b/pkg/apis/provisioning/v1alpha1/provisioner_status.go new file mode 100644 index 000000000000..8da72aef8a37 --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/provisioner_status.go @@ -0,0 +1,45 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +*/ + +package v1alpha1 + +import ( + "knative.dev/pkg/apis" +) + +// ProvisionerStatus defines the observed state of Provisioner +type ProvisionerStatus struct { + // LastScaleTime is the last time the Provisioner scaled the number + // of nodes + // +optional + LastScaleTime *apis.VolatileTime `json:"lastScaleTime,omitempty"` + + // Conditions is the set of conditions required for this provisioner to scale + // its target, and indicates whether or not those conditions are met. + // +optional + Conditions apis.Conditions `json:"conditions,omitempty"` +} + +func (p *Provisioner) StatusConditions() apis.ConditionManager { + return apis.NewLivingConditionSet( + Active, + ).Manage(p) +} + +func (s *Provisioner) GetConditions() apis.Conditions { + return s.Status.Conditions +} + +func (s *Provisioner) SetConditions(conditions apis.Conditions) { + s.Status.Conditions = conditions +} diff --git a/pkg/apis/provisioning/v1alpha1/provisioner_validation.go b/pkg/apis/provisioning/v1alpha1/provisioner_validation.go new file mode 100644 index 000000000000..7903011c1d4a --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/provisioner_validation.go @@ -0,0 +1,29 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1alpha1 + +import runtime "k8s.io/apimachinery/pkg/runtime" + +func (r *Provisioner) ValidateCreate() error { + return nil +} + +func (r *Provisioner) ValidateUpdate(old runtime.Object) error { + return nil +} + +func (r *Provisioner) ValidateDelete() error { + return nil +} diff --git a/pkg/apis/provisioning/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/provisioning/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 000000000000..992e9cb6051e --- /dev/null +++ b/pkg/apis/provisioning/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,182 @@ +// +build !ignore_autogenerated + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "knative.dev/pkg/apis" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AllocatorSpec) DeepCopyInto(out *AllocatorSpec) { + *out = *in + if in.InstanceTypes != nil { + in, out := &in.InstanceTypes, &out.InstanceTypes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocatorSpec. +func (in *AllocatorSpec) DeepCopy() *AllocatorSpec { + if in == nil { + return nil + } + out := new(AllocatorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. +func (in *ClusterSpec) DeepCopy() *ClusterSpec { + if in == nil { + return nil + } + out := new(ClusterSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Provisioner) DeepCopyInto(out *Provisioner) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Provisioner. +func (in *Provisioner) DeepCopy() *Provisioner { + if in == nil { + return nil + } + out := new(Provisioner) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Provisioner) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionerList) DeepCopyInto(out *ProvisionerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Provisioner, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerList. +func (in *ProvisionerList) DeepCopy() *ProvisionerList { + if in == nil { + return nil + } + out := new(ProvisionerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ProvisionerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionerSpec) DeepCopyInto(out *ProvisionerSpec) { + *out = *in + if in.Cluster != nil { + in, out := &in.Cluster, &out.Cluster + *out = new(ClusterSpec) + **out = **in + } + in.Allocator.DeepCopyInto(&out.Allocator) + out.Reallocator = in.Reallocator +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerSpec. +func (in *ProvisionerSpec) DeepCopy() *ProvisionerSpec { + if in == nil { + return nil + } + out := new(ProvisionerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProvisionerStatus) DeepCopyInto(out *ProvisionerStatus) { + *out = *in + if in.LastScaleTime != nil { + in, out := &in.LastScaleTime, &out.LastScaleTime + *out = new(apis.VolatileTime) + (*in).DeepCopyInto(*out) + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(apis.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerStatus. +func (in *ProvisionerStatus) DeepCopy() *ProvisionerStatus { + if in == nil { + return nil + } + out := new(ProvisionerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ReallocatorSpec) DeepCopyInto(out *ReallocatorSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReallocatorSpec. +func (in *ReallocatorSpec) DeepCopy() *ReallocatorSpec { + if in == nil { + return nil + } + out := new(ReallocatorSpec) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/autoscaler/algorithms/proportional.go b/pkg/autoscaler/algorithms/proportional.go index 3dc2be48a3ef..78da0c91aa9f 100644 --- a/pkg/autoscaler/algorithms/proportional.go +++ b/pkg/autoscaler/algorithms/proportional.go @@ -31,15 +31,15 @@ func (a *Proportional) GetDesiredReplicas(metric Metric, replicas int32) int32 { ratio := (metric.Metric.Value / metric.TargetValue) proportional := float64(replicas) * ratio switch metric.TargetType { - // Proportional + // Proportional, cannot scale to zero case v1alpha1.ValueMetricType: - return int32(math.Ceil(proportional)) + return int32(math.Max(1, math.Ceil(proportional))) // Proportional average, divided by number of replicas case v1alpha1.AverageValueMetricType: return int32(math.Ceil(ratio)) - // Proportional percentage, multiplied by 100 + // Proportional percentage, multiplied by 100, cannot scale to zero case v1alpha1.UtilizationMetricType: - return int32(math.Ceil(proportional * 100)) + return int32(math.Max(1, math.Ceil(proportional*100))) default: zap.S().Errorf("Unexpected TargetType %s for ", metric.TargetType) return replicas diff --git a/pkg/autoscaler/algorithms/proportional_test.go b/pkg/autoscaler/algorithms/proportional_test.go index 6094f70720d2..335ae6c72946 100644 --- a/pkg/autoscaler/algorithms/proportional_test.go +++ b/pkg/autoscaler/algorithms/proportional_test.go @@ -60,7 +60,7 @@ func TestProportionalGetDesiredReplicas(t *testing.T) { }, replicas: 0, }, - want: 0, + want: 1, }, { name: "AverageValueMetricType normal case", @@ -91,7 +91,7 @@ func TestProportionalGetDesiredReplicas(t *testing.T) { want: 7, }, { - name: "AverageUtilization normal case", + name: "Utilization normal case", args: args{ metric: Metric{ TargetType: v1alpha1.UtilizationMetricType, @@ -105,7 +105,7 @@ func TestProportionalGetDesiredReplicas(t *testing.T) { want: 3, }, { - name: "AverageUtilization does not scale to zero", + name: "Utilization does not scale to zero", args: args{ metric: Metric{ TargetType: v1alpha1.UtilizationMetricType, @@ -116,7 +116,7 @@ func TestProportionalGetDesiredReplicas(t *testing.T) { }, replicas: 0, }, - want: 0, + want: 1, }, { name: "Unknown metric type returns replicas", diff --git a/pkg/autoscaler/autoscaler.go b/pkg/autoscaler/autoscaler.go index 3bd20ed7fb10..0cde8fad6f57 100644 --- a/pkg/autoscaler/autoscaler.go +++ b/pkg/autoscaler/autoscaler.go @@ -90,7 +90,7 @@ func (a *Autoscaler) Reconcile() error { if err != nil { return err } - a.Status.CurrentReplicas = scaleTarget.Status.Replicas + a.Status.CurrentReplicas = &scaleTarget.Status.Replicas // 3. Calculate desired replicas using metrics and current desired replicas desiredReplicas := a.getDesiredReplicas(metrics, scaleTarget) @@ -107,7 +107,7 @@ func (a *Autoscaler) Reconcile() error { zap.S().With(zap.String("existing", fmt.Sprintf("%d", existingReplicas))). With(zap.String("desired", fmt.Sprintf("%d", desiredReplicas))). Info("Autoscaler scaled replicas count") - a.Status.DesiredReplicas = scaleTarget.Spec.Replicas + a.Status.DesiredReplicas = &scaleTarget.Spec.Replicas a.Status.LastScaleTime = &apis.VolatileTime{Inner: metav1.Now()} return nil } diff --git a/pkg/cloudprovider/aws/factory.go b/pkg/cloudprovider/aws/factory.go index be815457f9b5..2b7ea1519b5d 100644 --- a/pkg/cloudprovider/aws/factory.go +++ b/pkg/cloudprovider/aws/factory.go @@ -20,12 +20,17 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/autoscaling" "github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/eks/eksiface" + "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + provisioningv1alpha1 "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" "github.com/awslabs/karpenter/pkg/cloudprovider" + "github.com/awslabs/karpenter/pkg/cloudprovider/aws/fleet" "github.com/awslabs/karpenter/pkg/cloudprovider/fake" "github.com/awslabs/karpenter/pkg/utils/log" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,18 +38,23 @@ import ( type Factory struct { AutoscalingClient autoscalingiface.AutoScalingAPI - SQSClient sqsiface.SQSAPI - EKSClient eksiface.EKSAPI - Client client.Client + sqs sqsiface.SQSAPI + eks eksiface.EKSAPI + ec2 ec2iface.EC2API + fleetFactory *fleet.Factory + kubeClient client.Client } func NewFactory(options cloudprovider.Options) *Factory { sess := withRegion(session.Must(session.NewSession())) + EC2 := ec2.New(sess) return &Factory{ AutoscalingClient: autoscaling.New(sess), - EKSClient: eks.New(sess), - SQSClient: sqs.New(sess), - Client: options.Client, + eks: eks.New(sess), + sqs: sqs.New(sess), + ec2: EC2, + fleetFactory: fleet.NewFactory(EC2, iam.New(sess), options.Client), + kubeClient: options.Client, } } @@ -53,7 +63,7 @@ func (f *Factory) NodeGroupFor(spec *v1alpha1.ScalableNodeGroupSpec) cloudprovid case v1alpha1.AWSEC2AutoScalingGroup: return NewAutoScalingGroup(spec.ID, f.AutoscalingClient) case v1alpha1.AWSEKSNodeGroup: - return NewManagedNodeGroup(spec.ID, f.EKSClient, f.AutoscalingClient, f.Client) + return NewManagedNodeGroup(spec.ID, f.eks, f.AutoscalingClient, f.kubeClient) default: return fake.NewNotImplementedFactory().NodeGroupFor(spec) } @@ -62,12 +72,16 @@ func (f *Factory) NodeGroupFor(spec *v1alpha1.ScalableNodeGroupSpec) cloudprovid func (f *Factory) QueueFor(spec *v1alpha1.QueueSpec) cloudprovider.Queue { switch spec.Type { case v1alpha1.AWSSQSQueueType: - return NewSQSQueue(spec.ID, f.SQSClient) + return NewSQSQueue(spec.ID, f.sqs) default: return fake.NewNotImplementedFactory().QueueFor(spec) } } +func (f *Factory) CapacityFor(spec *provisioningv1alpha1.ProvisionerSpec) cloudprovider.Capacity { + return f.fleetFactory.For(spec) +} + func withRegion(sess *session.Session) *session.Session { region, err := ec2metadata.New(sess).Region() log.PanicIfError(err, "failed to call the metadata server's region API") diff --git a/pkg/cloudprovider/aws/fleet/capacity.go b/pkg/cloudprovider/aws/fleet/capacity.go new file mode 100644 index 000000000000..a6195229776a --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/capacity.go @@ -0,0 +1,101 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/cloudprovider" + "github.com/awslabs/karpenter/pkg/cloudprovider/aws/fleet/packing" + v1 "k8s.io/api/core/v1" +) + +// Capacity cloud provider implementation using AWS Fleet. +type Capacity struct { + spec *v1alpha1.ProvisionerSpec + nodeFactory *NodeFactory + packer packing.Packer + instanceProvider *InstanceProvider + vpcProvider *VPCProvider +} + +// Create a set of nodes given the constraints. +func (c *Capacity) Create(ctx context.Context, constraints *cloudprovider.Constraints) ([]cloudprovider.Packing, error) { + // 1. Compute Packing given the constraints + instancePackings, err := c.packer.Pack(ctx, constraints.Pods) + if err != nil { + return nil, fmt.Errorf("computing bin packing, %w", err) + } + + launchTemplate, err := c.vpcProvider.GetLaunchTemplate(ctx, c.spec.Cluster) + if err != nil { + return nil, fmt.Errorf("getting launch template, %w", err) + } + + zonalSubnetOptions, err := c.vpcProvider.GetZonalSubnets(ctx, constraints, c.spec.Cluster.Name) + if err != nil { + return nil, fmt.Errorf("getting zonal subnets, %w", err) + } + + // 2. Create Instances + var instanceIds []*string + podsForInstance := make(map[string][]*v1.Pod) + for _, packing := range instancePackings { + instanceID, err := c.instanceProvider.Create(ctx, launchTemplate, packing.InstanceTypeOptions, zonalSubnetOptions) + if err != nil { + // TODO Aggregate errors and continue + return nil, fmt.Errorf("creating capacity %w", err) + } + podsForInstance[*instanceID] = packing.Pods + instanceIds = append(instanceIds, instanceID) + } + + // 3. Convert to Nodes + nodes, err := c.nodeFactory.For(ctx, instanceIds) + if err != nil { + return nil, fmt.Errorf("determining nodes, %w", err) + } + nodePackings := []cloudprovider.Packing{} + for instanceID, node := range nodes { + nodePackings = append(nodePackings, cloudprovider.Packing{ + Node: node, + Pods: podsForInstance[instanceID], + }) + } + return nodePackings, nil +} + +// GetTopologyDomains returns a set of supported domains. +// e.g. us-west-2 -> [ us-west-2a, us-west-2b ] +func (c *Capacity) GetTopologyDomains(ctx context.Context, key cloudprovider.TopologyKey) ([]string, error) { + switch key { + case cloudprovider.TopologyKeyZone: + zones, err := c.vpcProvider.GetZones(ctx, c.spec.Cluster.Name) + if err != nil { + return nil, err + } + return zones, nil + case cloudprovider.TopologyKeySubnet: + subnets, err := c.vpcProvider.GetSubnetIds(ctx, c.spec.Cluster.Name) + if err != nil { + return nil, err + } + return subnets, nil + default: + return nil, fmt.Errorf("unrecognized topology key %s", key) + } +} diff --git a/pkg/cloudprovider/aws/fleet/factory.go b/pkg/cloudprovider/aws/fleet/factory.go new file mode 100644 index 000000000000..b57408455f4e --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/factory.go @@ -0,0 +1,84 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "time" + + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/cloudprovider/aws/fleet/packing" + "github.com/patrickmn/go-cache" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // CacheTTL restricts QPS to AWS APIs to this interval for verifying setup resources. + CacheTTL = 5 * time.Minute + // CacheCleanupInterval triggers cache cleanup (lazy eviction) at this interval. + CacheCleanupInterval = 10 * time.Minute + // ClusterTagKeyFormat is set on all Kubernetes owned resources. + ClusterTagKeyFormat = "kubernetes.io/cluster/%s" + // KarpenterTagKeyFormat is set on all Karpenter owned resources. + KarpenterTagKeyFormat = "karpenter.sh/cluster/%s" +) + +func NewFactory(ec2 ec2iface.EC2API, iam iamiface.IAMAPI, kubeClient client.Client) *Factory { + vpcProvider := &VPCProvider{ + launchTemplateProvider: &LaunchTemplateProvider{ + ec2: ec2, + launchTemplateCache: cache.New(CacheTTL, CacheCleanupInterval), + instanceProfileProvider: &InstanceProfileProvider{ + iam: iam, + kubeClient: kubeClient, + instanceProfileCache: cache.New(CacheTTL, CacheCleanupInterval), + }, + securityGroupProvider: &SecurityGroupProvider{ + ec2: ec2, + securityGroupCache: cache.New(CacheTTL, CacheCleanupInterval), + }, + }, + subnetProvider: &SubnetProvider{ + ec2: ec2, + subnetCache: cache.New(CacheTTL, CacheCleanupInterval), + }, + } + return &Factory{ + ec2: ec2, + vpcProvider: vpcProvider, + nodeFactory: &NodeFactory{ec2: ec2}, + instanceProvider: &InstanceProvider{ec2: ec2, vpc: vpcProvider}, + packer: packing.NewPacker(ec2), + } +} + +type Factory struct { + ec2 ec2iface.EC2API + vpcProvider *VPCProvider + nodeFactory *NodeFactory + instanceProvider *InstanceProvider + packer packing.Packer +} + +func (f *Factory) For(spec *v1alpha1.ProvisionerSpec) *Capacity { + return &Capacity{ + spec: spec, + nodeFactory: f.nodeFactory, + instanceProvider: f.instanceProvider, + vpcProvider: f.vpcProvider, + packer: f.packer, + } +} diff --git a/pkg/cloudprovider/aws/fleet/instance.go b/pkg/cloudprovider/aws/fleet/instance.go new file mode 100644 index 000000000000..4bb0e79ac4d0 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/instance.go @@ -0,0 +1,80 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + "math/rand" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" +) + +type InstanceProvider struct { + ec2 ec2iface.EC2API + vpc *VPCProvider +} + +// Create an instance given the constraints. +func (p *InstanceProvider) Create(ctx context.Context, + launchTemplate *ec2.LaunchTemplate, + instanceTypeOptions []string, + zonalSubnetOptions map[string][]*ec2.Subnet, +) (*string, error) { + // 1. Construct override options. + var overrides []*ec2.FleetLaunchTemplateOverridesRequest + for _, instanceType := range instanceTypeOptions { + for zone, subnets := range zonalSubnetOptions { + overrides = append(overrides, &ec2.FleetLaunchTemplateOverridesRequest{ + AvailabilityZone: aws.String(zone), + InstanceType: aws.String(instanceType), + // FleetAPI cannot span subnets from the same AZ, so randomize. + SubnetId: aws.String(*subnets[rand.Intn(len(subnets))].SubnetId), + }) + } + } + + // 2. Create fleet + createFleetOutput, err := p.ec2.CreateFleetWithContext(ctx, &ec2.CreateFleetInput{ + Type: aws.String(ec2.FleetTypeInstant), + TargetCapacitySpecification: &ec2.TargetCapacitySpecificationRequest{ + DefaultTargetCapacityType: aws.String(ec2.DefaultTargetCapacityTypeOnDemand), + TotalTargetCapacity: aws.Int64(1), + }, + LaunchTemplateConfigs: []*ec2.FleetLaunchTemplateConfigRequest{{ + LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{ + LaunchTemplateName: launchTemplate.LaunchTemplateName, + Version: aws.String("$Default"), + }, + Overrides: overrides, + }}, + }) + if err != nil { + return nil, fmt.Errorf("creating fleet %w", err) + } + // TODO aggregate errors + if count := len(createFleetOutput.Errors); count > 0 { + return nil, fmt.Errorf("errors while creating fleet, %v", createFleetOutput.Errors) + } + if count := len(createFleetOutput.Instances); count != 1 { + return nil, fmt.Errorf("expected 1 instance, but got %d", count) + } + if count := len(createFleetOutput.Instances[0].InstanceIds); count != 1 { + return nil, fmt.Errorf("expected 1 instance ids, but got %d", count) + } + return createFleetOutput.Instances[0].InstanceIds[0], nil +} diff --git a/pkg/cloudprovider/aws/fleet/instanceprofile.go b/pkg/cloudprovider/aws/fleet/instanceprofile.go new file mode 100644 index 000000000000..5f297672d767 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/instanceprofile.go @@ -0,0 +1,96 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/iam/iamiface" + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + KarpenterNodeInstanceProfileName = "KarpenterNodeInstanceProfile" +) + +type InstanceProfileProvider struct { + iam iamiface.IAMAPI + kubeClient client.Client + instanceProfileCache *cache.Cache +} + +func NewInstanceProfileProvider(iam iamiface.IAMAPI, kubeClient client.Client) *InstanceProfileProvider { + return &InstanceProfileProvider{ + iam: iam, + kubeClient: kubeClient, + instanceProfileCache: cache.New(CacheTTL, CacheCleanupInterval), + } +} + +func (p *InstanceProfileProvider) Get(ctx context.Context, cluster *v1alpha1.ClusterSpec) (*iam.InstanceProfile, error) { + if instanceProfile, ok := p.instanceProfileCache.Get(cluster.Name); ok { + return instanceProfile.(*iam.InstanceProfile), nil + } + return p.getInstanceProfile(ctx, cluster) +} + +func (p *InstanceProfileProvider) getInstanceProfile(ctx context.Context, cluster *v1alpha1.ClusterSpec) (*iam.InstanceProfile, error) { + output, err := p.iam.GetInstanceProfileWithContext(ctx, &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(KarpenterNodeInstanceProfileName), + }) + if err != nil { + return nil, fmt.Errorf("retriving instance profile %s, %w", IAMInstanceProfileName, err) + } + for _, role := range output.InstanceProfile.Roles { + if err := p.addToAWSAuthConfigmap(role); err != nil { + return nil, fmt.Errorf("adding role %s, %w", *role.RoleName, err) + } + } + zap.S().Debugf("Successfully discovered instance profile %s for cluster %s", *output.InstanceProfile.InstanceProfileName, cluster.Name) + p.instanceProfileCache.Set(cluster.Name, output.InstanceProfile, CacheTTL) + return output.InstanceProfile, nil +} + +func (p *InstanceProfileProvider) addToAWSAuthConfigmap(role *iam.Role) error { + awsAuth := &v1.ConfigMap{} + if err := p.kubeClient.Get(context.TODO(), types.NamespacedName{Name: "aws-auth", Namespace: "kube-system"}, awsAuth); err != nil { + return fmt.Errorf("retrieving configmap aws-auth, %w", err) + } + if strings.Contains(awsAuth.Data["mapRoles"], *role.Arn) { + zap.S().Debugf("Successfully detected aws-auth configmap contains roleArn %s", *role.Arn) + return nil + } + // Since the aws-auth configmap is stringly typed, this specific indentation is critical + awsAuth.Data["mapRoles"] += fmt.Sprintf(` +- groups: + - system:bootstrappers + - system:nodes + rolearn: %s + username: system:node:{{EC2PrivateDNSName}}`, *role.Arn) + if err := p.kubeClient.Update(context.TODO(), awsAuth); err != nil { + return fmt.Errorf("updating configmap aws-auth, %w", err) + } + zap.S().Debugf("Successfully patched configmap aws-auth with roleArn %s", *role.Arn) + return nil +} diff --git a/pkg/cloudprovider/aws/fleet/launchtemplate.go b/pkg/cloudprovider/aws/fleet/launchtemplate.go new file mode 100644 index 000000000000..8fef6e9f8480 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/launchtemplate.go @@ -0,0 +1,131 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "encoding/base64" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" +) + +const ( + LaunchTemplateNameFormat = "Karpenter-%s" + IAMInstanceProfileName = "KarpenterNodeRole" +) + +type LaunchTemplateProvider struct { + ec2 ec2iface.EC2API + launchTemplateCache *cache.Cache + instanceProfileProvider *InstanceProfileProvider + securityGroupProvider *SecurityGroupProvider +} + +func (p *LaunchTemplateProvider) Get(ctx context.Context, cluster *v1alpha1.ClusterSpec) (*ec2.LaunchTemplate, error) { + if launchTemplate, ok := p.launchTemplateCache.Get(cluster.Name); ok { + return launchTemplate.(*ec2.LaunchTemplate), nil + } + launchTemplate, err := p.getLaunchTemplate(ctx, cluster) + if err != nil { + return nil, err + } + p.launchTemplateCache.Set(cluster.Name, launchTemplate, CacheTTL) + return launchTemplate, nil +} + +// TODO, reconcile launch template if not equal to desired launch template (AMI upgrade, role changed, etc) +func (p *LaunchTemplateProvider) getLaunchTemplate(ctx context.Context, cluster *v1alpha1.ClusterSpec) (*ec2.LaunchTemplate, error) { + describelaunchTemplateOutput, err := p.ec2.DescribeLaunchTemplatesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ + LaunchTemplateNames: []*string{aws.String(fmt.Sprintf(LaunchTemplateNameFormat, cluster.Name))}, + }) + if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "InvalidLaunchTemplateName.NotFoundException" { + return p.createLaunchTemplate(ctx, cluster) + } + if err != nil { + return nil, fmt.Errorf("describing launch templates, %w", err) + } + if length := len(describelaunchTemplateOutput.LaunchTemplates); length > 1 { + return nil, fmt.Errorf("expected to find one launch template, but found %d", length) + } + launchTemplate := describelaunchTemplateOutput.LaunchTemplates[0] + zap.S().Debugf("Successfully discovered launch template %s for cluster %s", *launchTemplate.LaunchTemplateName, cluster.Name) + return launchTemplate, nil +} + +func (p *LaunchTemplateProvider) createLaunchTemplate(ctx context.Context, cluster *v1alpha1.ClusterSpec) (*ec2.LaunchTemplate, error) { + securityGroupIds, err := p.getSecurityGroupIds(ctx, cluster) + if err != nil { + return nil, fmt.Errorf("getting security groups, %w", err) + } + + instanceProfile, err := p.instanceProfileProvider.Get(ctx, cluster) + if err != nil { + return nil, fmt.Errorf("getting instance profile, %w", err) + } + + output, err := p.ec2.CreateLaunchTemplate(&ec2.CreateLaunchTemplateInput{ + LaunchTemplateName: aws.String(fmt.Sprintf(LaunchTemplateNameFormat, cluster.Name)), + LaunchTemplateData: &ec2.RequestLaunchTemplateData{ + IamInstanceProfile: &ec2.LaunchTemplateIamInstanceProfileSpecificationRequest{ + Name: instanceProfile.InstanceProfileName, + }, + TagSpecifications: []*ec2.LaunchTemplateTagSpecificationRequest{{ + ResourceType: aws.String(ec2.ResourceTypeInstance), + Tags: []*ec2.Tag{{ + Key: aws.String(fmt.Sprintf(ClusterTagKeyFormat, cluster.Name)), + Value: aws.String("owned"), + }}, + }}, + SecurityGroupIds: securityGroupIds, + UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(` + #!/bin/bash + yum install -y https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/linux_amd64/amazon-ssm-agent.rpm + /etc/eks/bootstrap.sh %s \ + --kubelet-extra-args '--node-labels=karpenter.sh/provisioned=true' \ + --b64-cluster-ca %s \ + --apiserver-endpoint %s`, + cluster.Name, + cluster.CABundle, + cluster.Endpoint, + )))), + // TODO discover this with SSM + ImageId: aws.String("ami-0532808ed453f9ca3"), + }, + }) + if err != nil { + return nil, fmt.Errorf("creating launch template, %w", err) + } + zap.S().Debugf("Successfully created default launch template, %s", *output.LaunchTemplate.LaunchTemplateName) + return output.LaunchTemplate, nil +} + +func (p *LaunchTemplateProvider) getSecurityGroupIds(ctx context.Context, cluster *v1alpha1.ClusterSpec) ([]*string, error) { + securityGroupIds := []*string{} + securityGroups, err := p.securityGroupProvider.Get(ctx, cluster.Name) + if err != nil { + return nil, err + } + for _, securityGroup := range securityGroups { + securityGroupIds = append(securityGroupIds, securityGroup.GroupId) + } + return securityGroupIds, nil +} diff --git a/pkg/cloudprovider/aws/fleet/nodefactory.go b/pkg/cloudprovider/aws/fleet/nodefactory.go new file mode 100644 index 000000000000..20e8454413a4 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/nodefactory.go @@ -0,0 +1,85 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "go.uber.org/zap" + "gopkg.in/retry.v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type NodeFactory struct { + ec2 ec2iface.EC2API +} + +// For a given set of instanceIds return a map of instanceID to Kubernetes node object. +func (n *NodeFactory) For(ctx context.Context, instanceIds []*string) (map[string]*v1.Node, error) { + // Backoff retry is necessary here because EC2's APIs are eventually + // consistent. In most cases, this call will only be made once. + // TODO Use https://docs.aws.amazon.com/sdk-for-go/api/aws/request/#WithRetryer + for attempt := retry.Start(retry.Exponential{ + Initial: 1 * time.Second, + MaxDelay: 10 * time.Second, + Factor: 2, Jitter: true, + }, nil); attempt.Next(); { + describeInstancesOutput, err := n.ec2.DescribeInstances(&ec2.DescribeInstancesInput{InstanceIds: instanceIds}) + if err == nil { + return n.nodesFrom(describeInstancesOutput.Reservations), nil + } + if aerr, ok := err.(awserr.Error); ok && aerr.Code() != "InvalidInstanceID.NotFound" { + return nil, aerr + } + zap.S().Debugf("Retrying DescribeInstances due to eventual consistency: fleet created, but instances not yet found.") + } + return nil, fmt.Errorf("failed to describe ec2 instances") +} + +func (n *NodeFactory) nodesFrom(reservations []*ec2.Reservation) map[string]*v1.Node { + nodes := map[string]*v1.Node{} + for _, reservation := range reservations { + for _, instance := range reservation.Instances { + nodes[*instance.InstanceId] = n.nodeFrom(instance) + } + } + return nodes +} + +func (n *NodeFactory) nodeFrom(instance *ec2.Instance) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: *instance.PrivateDnsName, + }, + Spec: v1.NodeSpec{ + ProviderID: fmt.Sprintf("aws:///%s/%s", *instance.Placement.AvailabilityZone, *instance.InstanceId), + }, + Status: v1.NodeStatus{ + Allocatable: v1.ResourceList{ + // TODO, This value is necessary to avoid OutOfPods failure state. Find a way to set this (and cpu/mem) correctly + v1.ResourcePods: resource.MustParse("100"), + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + } +} diff --git a/pkg/cloudprovider/aws/fleet/packing/packing.go b/pkg/cloudprovider/aws/fleet/packing/packing.go new file mode 100644 index 000000000000..3c46698bc895 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/packing/packing.go @@ -0,0 +1,56 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package packing + +import ( + "context" + + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" +) + +type PodPacker struct { + ec2 ec2iface.EC2API +} + +// PodPacker helps pack the pods and calculates efficient placement on the instances. +type Packer interface { + Pack(ctx context.Context, pods []*v1.Pod) ([]*Packings, error) +} + +// Packings contains a list of pods that can be placed on any of Instance type +// in the InstanceTypeOptions +type Packings struct { + Pods []*v1.Pod + InstanceTypeOptions []string +} + +func NewPacker(ec2 ec2iface.EC2API) *PodPacker { + return &PodPacker{ec2: ec2} +} + +// Pack returns the packings for the provided pods. Computes a set of viable +// instance types for each packing of pods. Instance variety enables EC2 to make +// better cost and availability decisions. +func (p *PodPacker) Pack(ctx context.Context, pods []*v1.Pod) ([]*Packings, error) { + zap.S().Debugf("Successfully packed %d pods onto %d nodes", len(pods), 1) + return []*Packings{ + { + InstanceTypeOptions: []string{"m5.large"}, // TODO, prioritize possible instance types + Pods: pods, + }, + }, nil +} diff --git a/pkg/cloudprovider/aws/fleet/securitygroups.go b/pkg/cloudprovider/aws/fleet/securitygroups.go new file mode 100644 index 000000000000..c56a43ed7fab --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/securitygroups.go @@ -0,0 +1,62 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" +) + +type SecurityGroupProvider struct { + ec2 ec2iface.EC2API + securityGroupCache *cache.Cache +} + +func NewSecurityGroupProvider(ec2 ec2iface.EC2API) *SecurityGroupProvider { + return &SecurityGroupProvider{ + ec2: ec2, + securityGroupCache: cache.New(CacheTTL, CacheCleanupInterval), + } +} + +func (s *SecurityGroupProvider) Get(ctx context.Context, clusterName string) ([]*ec2.SecurityGroup, error) { + if securityGroups, ok := s.securityGroupCache.Get(clusterName); ok { + return securityGroups.([]*ec2.SecurityGroup), nil + } + return s.getSecurityGroups(ctx, clusterName) +} + +func (s *SecurityGroupProvider) getSecurityGroups(ctx context.Context, clusterName string) ([]*ec2.SecurityGroup, error) { + describeSecurityGroupOutput, err := s.ec2.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{ + Filters: []*ec2.Filter{{ + Name: aws.String("tag-key"), + Values: []*string{aws.String(fmt.Sprintf(ClusterTagKeyFormat, clusterName))}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("describing security groups with tag key %s, %w", fmt.Sprintf(ClusterTagKeyFormat, clusterName), err) + } + + securityGroups := describeSecurityGroupOutput.SecurityGroups + s.securityGroupCache.Set(clusterName, securityGroups, CacheTTL) + zap.S().Debugf("Successfully discovered %d security groups for cluster %s", len(securityGroups), clusterName) + return securityGroups, nil +} diff --git a/pkg/cloudprovider/aws/fleet/vpc.go b/pkg/cloudprovider/aws/fleet/vpc.go new file mode 100644 index 000000000000..4f7d0dc5f256 --- /dev/null +++ b/pkg/cloudprovider/aws/fleet/vpc.go @@ -0,0 +1,158 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fleet + +import ( + "context" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/cloudprovider" + "github.com/patrickmn/go-cache" + "go.uber.org/zap" +) + +type VPCProvider struct { + launchTemplateProvider *LaunchTemplateProvider + subnetProvider *SubnetProvider +} + +func (p *VPCProvider) GetLaunchTemplate(ctx context.Context, clusterSpec *v1alpha1.ClusterSpec) (*ec2.LaunchTemplate, error) { + return p.launchTemplateProvider.Get(ctx, clusterSpec) +} + +func (p *VPCProvider) GetZones(ctx context.Context, clusterName string) ([]string, error) { + zonalSubnets, err := p.subnetProvider.Get(ctx, clusterName) + if err != nil { + return nil, err + } + zones := []string{} + for zone := range zonalSubnets { + zones = append(zones, zone) + } + return zones, nil +} + +func (p *VPCProvider) GetZonalSubnets(ctx context.Context, constraints *cloudprovider.Constraints, clusterName string) (map[string][]*ec2.Subnet, error) { + // 1. Get all subnets + zonalSubnets, err := p.subnetProvider.Get(ctx, clusterName) + if err != nil { + return nil, fmt.Errorf("getting zonal subnets, %w", err) + } + // 2. Return specific subnet if specified. + if subnetID, ok := constraints.Topology[cloudprovider.TopologyKeySubnet]; ok { + for zone, subnets := range zonalSubnets { + for _, subnet := range subnets { + if subnetID == *subnet.SubnetId { + return map[string][]*ec2.Subnet{zone: {subnet}}, nil + } + } + } + return nil, fmt.Errorf("no subnet exists named %s", subnetID) + } + // 3. Constrain by zones + constrainedZones, err := p.getConstrainedZones(ctx, constraints, clusterName) + if err != nil { + return nil, fmt.Errorf("getting zones, %w", err) + } + constrainedZonalSubnets := map[string][]*ec2.Subnet{} + for zone, subnets := range zonalSubnets { + for _, constrainedZone := range constrainedZones { + if zone == constrainedZone { + constrainedZonalSubnets[constrainedZone] = subnets + } + } + } + if len(constrainedZonalSubnets) == 0 { + return nil, fmt.Errorf("failed to find viable zonal subnet pairing") + } + return constrainedZonalSubnets, nil +} + +func (p *VPCProvider) GetSubnetIds(ctx context.Context, clusterName string) ([]string, error) { + zonalSubnets, err := p.subnetProvider.Get(ctx, clusterName) + if err != nil { + return nil, err + } + subnetIds := []string{} + for _, subnets := range zonalSubnets { + for _, subnet := range subnets { + subnetIds = append(subnetIds, *subnet.SubnetId) + } + } + return subnetIds, nil +} + +func (p *VPCProvider) getConstrainedZones(ctx context.Context, constraints *cloudprovider.Constraints, clusterName string) ([]string, error) { + // 1. Return zone if specified. + if zone, ok := constraints.Topology[cloudprovider.TopologyKeyZone]; ok { + return []string{zone}, nil + } + // 2. Return all zone options + zones, err := p.GetZones(ctx, clusterName) + if err != nil { + return nil, err + } + return zones, nil +} + +type ZonalSubnets map[string][]*ec2.Subnet + +type SubnetProvider struct { + ec2 ec2iface.EC2API + subnetCache *cache.Cache +} + +func NewSubnetProvider(ec2 ec2iface.EC2API) *SubnetProvider { + return &SubnetProvider{ + ec2: ec2, + subnetCache: cache.New(CacheTTL, CacheCleanupInterval), + } +} + +func (s *SubnetProvider) Get(ctx context.Context, clusterName string) (ZonalSubnets, error) { + if zonalSubnets, ok := s.subnetCache.Get(clusterName); ok { + return zonalSubnets.(ZonalSubnets), nil + } + return s.getZonalSubnets(ctx, clusterName) +} + +func (s *SubnetProvider) getZonalSubnets(ctx context.Context, clusterName string) (ZonalSubnets, error) { + describeSubnetOutput, err := s.ec2.DescribeSubnetsWithContext(ctx, &ec2.DescribeSubnetsInput{ + Filters: []*ec2.Filter{{ + Name: aws.String("tag-key"), + Values: []*string{aws.String(fmt.Sprintf(ClusterTagKeyFormat, clusterName))}, + }}, + }) + if err != nil { + return nil, fmt.Errorf("describing subnets, %w", err) + } + + zonalSubnetMap := ZonalSubnets{} + for _, subnet := range describeSubnetOutput.Subnets { + if subnets, ok := zonalSubnetMap[*subnet.AvailabilityZone]; ok { + zonalSubnetMap[*subnet.AvailabilityZone] = append(subnets, subnet) + } else { + zonalSubnetMap[*subnet.AvailabilityZone] = []*ec2.Subnet{subnet} + } + } + + s.subnetCache.Set(clusterName, zonalSubnetMap, CacheTTL) + zap.S().Infof("Successfully discovered subnets in %d zones for cluster %s", len(zonalSubnetMap), clusterName) + return zonalSubnetMap, nil +} diff --git a/pkg/cloudprovider/fake/capacity.go b/pkg/cloudprovider/fake/capacity.go new file mode 100644 index 000000000000..b68bdefa30d9 --- /dev/null +++ b/pkg/cloudprovider/fake/capacity.go @@ -0,0 +1,32 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "context" + + "github.com/awslabs/karpenter/pkg/cloudprovider" +) + +type Capacity struct { +} + +func (c *Capacity) Create(ctx context.Context, constraints *cloudprovider.Constraints) ([]cloudprovider.Packing, error) { + return nil, nil +} + +func (c *Capacity) GetTopologyDomains(ctx context.Context, key cloudprovider.TopologyKey) ([]string, error) { + return nil, nil +} diff --git a/pkg/cloudprovider/fake/errors.go b/pkg/cloudprovider/fake/errors.go new file mode 100644 index 000000000000..b3d53d4da252 --- /dev/null +++ b/pkg/cloudprovider/fake/errors.go @@ -0,0 +1,32 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +// retryableErr implements controllers.RetryableError & controllers.CodedError +type retryableErr struct { + error + retryable bool +} + +func (e *retryableErr) IsRetryable() bool { + return e.retryable +} +func (e *retryableErr) ErrorCode() string { + return e.Error() +} + +func RetryableError(err error) *retryableErr { + return &retryableErr{error: err, retryable: true} +} diff --git a/pkg/cloudprovider/fake/factory.go b/pkg/cloudprovider/fake/factory.go index 549f28fd62cb..8e02411ce409 100644 --- a/pkg/cloudprovider/fake/factory.go +++ b/pkg/cloudprovider/fake/factory.go @@ -18,23 +18,29 @@ import ( "fmt" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + provisioningv1alpha1 "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" "github.com/awslabs/karpenter/pkg/cloudprovider" - "knative.dev/pkg/ptr" ) var ( NotImplementedError = fmt.Errorf("provider is not implemented. Are you running the correct release for your cloud provider?") ) +const ( + NodeGroupMessage = "fake factory message" +) + type Factory struct { WantErr error - // NodeReplicas is use by tests to control observed replicas. - NodeReplicas map[string]int32 + // NodeReplicas is used by tests to control observed replicas. + NodeReplicas map[string]*int32 + NodeGroupStable bool } func NewFactory(options cloudprovider.Options) *Factory { return &Factory{ - NodeReplicas: make(map[string]int32), + NodeReplicas: make(map[string]*int32), + NodeGroupStable: true, } } @@ -43,12 +49,21 @@ func NewNotImplementedFactory() *Factory { } func (f *Factory) NodeGroupFor(sng *v1alpha1.ScalableNodeGroupSpec) cloudprovider.NodeGroup { + msg := "" + if !f.NodeGroupStable { + msg = NodeGroupMessage + } return &NodeGroup{ WantErr: f.WantErr, - Replicas: ptr.Int32(f.NodeReplicas[sng.ID]), + Stable: f.NodeGroupStable, + Message: msg, + Replicas: f.NodeReplicas[sng.ID], } } func (f *Factory) QueueFor(spec *v1alpha1.QueueSpec) cloudprovider.Queue { return &Queue{Id: spec.ID, WantErr: f.WantErr} } +func (f *Factory) CapacityFor(spec *provisioningv1alpha1.ProvisionerSpec) cloudprovider.Capacity { + return &Capacity{} +} diff --git a/pkg/cloudprovider/fake/nodegroup.go b/pkg/cloudprovider/fake/nodegroup.go index 31d991b648ba..6b38eadc5618 100644 --- a/pkg/cloudprovider/fake/nodegroup.go +++ b/pkg/cloudprovider/fake/nodegroup.go @@ -19,6 +19,8 @@ import ( type NodeGroup struct { WantErr error + Stable bool + Message string Replicas *int32 } @@ -41,5 +43,5 @@ func (n *NodeGroup) SetReplicas(count int32) error { } func (n *NodeGroup) Stabilized() (bool, string, error) { - return true, "", nil + return n.Stable, n.Message, nil } diff --git a/pkg/cloudprovider/types.go b/pkg/cloudprovider/types.go index 848bd394eb21..92dca5a46492 100644 --- a/pkg/cloudprovider/types.go +++ b/pkg/cloudprovider/types.go @@ -15,16 +15,22 @@ limitations under the License. package cloudprovider import ( - "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + "context" + + autoscalingv1alpha1 "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + provisioningv1alpha1 "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) // Factory instantiates the cloud provider's resources type Factory interface { // NodeGroupFor returns a node group for the provided spec - NodeGroupFor(sng *v1alpha1.ScalableNodeGroupSpec) NodeGroup + NodeGroupFor(sng *autoscalingv1alpha1.ScalableNodeGroupSpec) NodeGroup // QueueFor returns a queue for the provided spec - QueueFor(queue *v1alpha1.QueueSpec) Queue + QueueFor(queue *autoscalingv1alpha1.QueueSpec) Queue + // Capacity returns a provisioner for the provider to create instances + CapacityFor(spec *provisioningv1alpha1.ProvisionerSpec) Capacity } // Queue abstracts all provider specific behavior for Queues @@ -49,6 +55,56 @@ type NodeGroup interface { Stabilized() (bool, string, error) } +// Capacity provisions a set of nodes that fulfill a set of constraints. +type Capacity interface { + // Create a set of nodes to fulfill the desired capacity given constraints. + Create(context.Context, *Constraints) ([]Packing, error) + + // GetTopologyDomains returns a list of topology domains supported by the + // cloud provider for the given key. + // For example, GetTopologyDomains("zone") -> [ "us-west-2a", "us-west-2b" ] + // This enables the caller to to build Constraints for a known set of + GetTopologyDomains(context.Context, TopologyKey) ([]string, error) +} + +// Constraints lets the controller define the desired capacity, +// avalability zone, architecture for the desired nodes. +type Constraints struct { + // Pods is a list of equivalently schedulable pods to be efficiently + // binpacked. + Pods []*v1.Pod + // Overhead resources per node from system resources such a kubelet and + // daemonsets. + Overhead v1.ResourceList + // Topology constrains the topology of the node, e.g. "zone". + Topology map[TopologyKey]string + // Architecture constrains the underlying architecture. + Architecture Architecture +} + +// Packing is a solution to packing pods onto nodes given constraints. +type Packing struct { + Node *v1.Node + Pods []*v1.Pod +} + +// TopologyKey: +// https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/ +type TopologyKey string + +const ( + TopologyKeyZone TopologyKey = "zone" + TopologyKeySubnet TopologyKey = "subnet" +) + +// Architecture constrains the underlying node's compilation architecture. +type Architecture string + +const ( + ArchitectureLinux386 Architecture = "linux/386" + // LinuxAMD64 Architecture = "linux/amd64" TODO +) + // Options are injected into cloud providers' factories type Options struct { Client client.Client diff --git a/pkg/controllers/controller.go b/pkg/controllers/controller.go index bd8b6a39d19e..eb4d825ee757 100644 --- a/pkg/controllers/controller.go +++ b/pkg/controllers/controller.go @@ -17,6 +17,7 @@ package controllers import ( "context" "fmt" + "sigs.k8s.io/controller-runtime/pkg/webhook" "time" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" @@ -46,10 +47,20 @@ type Controller interface { Owns() []Object } +// NamedController allows controllers to optionally implement a Name() function which will be used instead of the +// reconciled resource's name. This is useful when writing multiple controllers for a single resource type. +type NamedController interface { + Controller + // Name returns the name of the controller + Name() string +} + // Object provides an abstraction over a kubernetes custom resource with // methods necessary to standardize reconciliation behavior in Karpenter. type Object interface { client.Object + webhook.Validator + webhook.Defaulter StatusConditions() apis.ConditionManager } @@ -72,15 +83,21 @@ func (c *GenericController) Reconcile(ctx context.Context, req reconcile.Request } // 2. Copy object for merge patch base persisted := resource.DeepCopyObject() - // 3. Reconcile - if err := c.Controller.Reconcile(resource); err != nil { + // 3. Validate + if err := c.For().ValidateCreate(); err != nil { + resource.StatusConditions().MarkFalse(v1alpha1.Active, "could not validate kind: %v err: %v", + resource.GetObjectKind().GroupVersionKind().Kind, err.Error()) + zap.S().Errorf("Controller failed to validate kind: %v err: %v", + resource.GetObjectKind().GroupVersionKind().Kind, err) + // 4. Reconcile + } else if err := c.Controller.Reconcile(resource); err != nil { resource.StatusConditions().MarkFalse(v1alpha1.Active, "", err.Error()) zap.S().Errorf("Controller failed to reconcile kind: %v err: %v", resource.GetObjectKind().GroupVersionKind().Kind, err) } else { resource.StatusConditions().MarkTrue(v1alpha1.Active) } - // 4. Update Status using a merge patch + // 5. Update Status using a merge patch if err := c.Status().Patch(ctx, resource, client.MergeFrom(persisted)); err != nil { return reconcile.Result{}, fmt.Errorf("Failed to persist changes to %s, %w", req.NamespacedName, err) } diff --git a/pkg/controllers/horizontalautoscaler/v1alpha1/suite_test.go b/pkg/controllers/horizontalautoscaler/v1alpha1/suite_test.go index b10288121cdc..b104e1908626 100644 --- a/pkg/controllers/horizontalautoscaler/v1alpha1/suite_test.go +++ b/pkg/controllers/horizontalautoscaler/v1alpha1/suite_test.go @@ -81,6 +81,9 @@ var _ = Describe("Examples", func() { Expect(err).NotTo(HaveOccurred()) ha = &v1alpha1.HorizontalAutoscaler{} sng = &v1alpha1.ScalableNodeGroup{} + v1alpha1.RegisterScalableNodeGroupValidator(v1alpha1.AWSEKSNodeGroup, func(sng *v1alpha1.ScalableNodeGroupSpec) error { + return nil + }) }) AfterEach(func() { @@ -91,12 +94,12 @@ var _ = Describe("Examples", func() { It("should scale to average utilization target, metric=85, target=60, replicas=5, want=8", func() { Expect(ns.ParseResources("docs/examples/reserved-capacity-utilization.yaml", ha, sng)).To(Succeed()) sng.Spec.Replicas = ptr.Int32(5) - fakeCloudProvider.NodeReplicas[sng.Spec.ID] = *sng.Spec.Replicas + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = ptr.Int32(*sng.Spec.Replicas) // create a new pointer to avoid races with the controller MockMetricValue(fakeServer, .85) ExpectCreated(ns.Client, sng, ha) ExpectEventuallyHappy(ns.Client, sng, ha) - Expect(ha.Status.DesiredReplicas).To(BeEquivalentTo(8), log.Pretty(ha)) + Expect(*ha.Status.DesiredReplicas).To(BeEquivalentTo(8), log.Pretty(ha)) ExpectDeleted(ns.Client, ha) }) }) @@ -105,12 +108,12 @@ var _ = Describe("Examples", func() { It("should scale to average value target, metric=41, target=4, want=11", func() { Expect(ns.ParseResources("docs/examples/queue-length-average-value.yaml", ha, sng)).To(Succeed()) sng.Spec.Replicas = ptr.Int32(1) - fakeCloudProvider.NodeReplicas[sng.Spec.ID] = *sng.Spec.Replicas + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = ptr.Int32(*sng.Spec.Replicas) // create a new pointer to avoid races with the controller MockMetricValue(fakeServer, 41) ExpectCreated(ns.Client, sng, ha) ExpectEventuallyHappy(ns.Client, sng, ha) - Expect(ha.Status.DesiredReplicas).To(BeEquivalentTo(11), log.Pretty(ha)) + Expect(*ha.Status.DesiredReplicas).To(BeEquivalentTo(11), log.Pretty(ha)) ExpectDeleted(ns.Client, ha) }) }) diff --git a/pkg/controllers/manager.go b/pkg/controllers/manager.go index a61b588e2fdb..e28fcdff023f 100644 --- a/pkg/controllers/manager.go +++ b/pkg/controllers/manager.go @@ -16,7 +16,6 @@ package controllers import ( "context" - "github.com/awslabs/karpenter/pkg/apis" "github.com/awslabs/karpenter/pkg/utils/log" v1 "k8s.io/api/core/v1" @@ -58,14 +57,18 @@ func NewManagerOrDie(config *rest.Config, options controllerruntime.Options) Man func (m *GenericControllerManager) Register(controllers ...Controller) Manager { for _, controller := range controllers { - var builder = controllerruntime.NewControllerManagedBy(m).For(controller.For()) + controlledObject := controller.For() + var builder = controllerruntime.NewControllerManagedBy(m).For(controlledObject) + if namedController, ok := controller.(NamedController); ok { + builder.Named(namedController.Name()) + } for _, resource := range controller.Owns() { builder = builder.Owns(resource) } log.PanicIfError(builder.Complete(&GenericController{Controller: controller, Client: m.GetClient()}), - "Failed to register controller to manager for %s", controller.For()) - log.PanicIfError(controllerruntime.NewWebhookManagedBy(m).For(controller.For()).Complete(), - "Failed to register controller to manager for %s", controller.For()) + "Failed to register controller to manager for %s", controlledObject) + log.PanicIfError(controllerruntime.NewWebhookManagedBy(m).For(controlledObject).Complete(), + "Failed to register controller to manager for %s", controlledObject) } return m } diff --git a/pkg/controllers/metricsproducer/v1alpha1/suite_test.go b/pkg/controllers/metricsproducer/v1alpha1/suite_test.go index c03c99d24c90..e48376308569 100644 --- a/pkg/controllers/metricsproducer/v1alpha1/suite_test.go +++ b/pkg/controllers/metricsproducer/v1alpha1/suite_test.go @@ -66,28 +66,28 @@ var _ = Describe("Examples", func() { Expect(ns.ParseResources("docs/examples/reserved-capacity-utilization.yaml", mp)).To(Succeed()) mp.Spec.ReservedCapacity.NodeSelector = map[string]string{"k8s.io/nodegroup": ns.Name} - capacity := v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("16"), - v1.ResourceMemory: resource.MustParse("128Gi"), + allocatable := v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("16300m"), + v1.ResourceMemory: resource.MustParse("128500Mi"), v1.ResourcePods: resource.MustParse("50"), } nodes := []client.Object{ - test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Capacity: capacity}), - test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Capacity: capacity}), - test.NodeWith(test.NodeOptions{Labels: map[string]string{"unknown": "label"}, Capacity: capacity}), - test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Capacity: capacity}), - test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Capacity: capacity, ReadyStatus: v1.ConditionFalse}), - test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Capacity: capacity, Unschedulable: true}), + test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Allocatable: allocatable}), + test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Allocatable: allocatable}), + test.NodeWith(test.NodeOptions{Labels: map[string]string{"unknown": "label"}, Allocatable: allocatable}), + test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Allocatable: allocatable}), + test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Allocatable: allocatable, ReadyStatus: v1.ConditionFalse}), + test.NodeWith(test.NodeOptions{Labels: mp.Spec.ReservedCapacity.NodeSelector, Allocatable: allocatable, Unschedulable: true}), } pods := []client.Object{ // node[0] 6/16 cores, 76/128 gig allocated - test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}), - test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("2"), v1.ResourceMemory: resource.MustParse("25Gi")}), - test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("3"), v1.ResourceMemory: resource.MustParse("50Gi")}), + test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("1100m"), v1.ResourceMemory: resource.MustParse("1Gi")}), + test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("2100m"), v1.ResourceMemory: resource.MustParse("25Gi")}), + test.Pod(nodes[0].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("3300m"), v1.ResourceMemory: resource.MustParse("50Gi")}), // node[1] 1/16 cores, 76/128 gig allocated - test.Pod(nodes[1].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("1Gi")}), + test.Pod(nodes[1].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("1100m"), v1.ResourceMemory: resource.MustParse("1Gi")}), // node[2] is ignored test.Pod(nodes[2].GetName(), ns.Name, v1.ResourceList{v1.ResourceCPU: resource.MustParse("99"), v1.ResourceMemory: resource.MustParse("99Gi")}), // node[3] is unallocated @@ -100,9 +100,9 @@ var _ = Describe("Examples", func() { ExpectCreated(ns.Client, mp) ExpectEventuallyHappy(ns.Client, mp) - Expect(mp.Status.ReservedCapacity[v1.ResourceCPU]).To(BeEquivalentTo("14%, 7/48")) - Expect(mp.Status.ReservedCapacity[v1.ResourceMemory]).To(BeEquivalentTo("20%, 82678120448/412316860416")) - Expect(mp.Status.ReservedCapacity[v1.ResourcePods]).To(BeEquivalentTo("2%, 4/150")) + Expect(mp.Status.ReservedCapacity[v1.ResourceCPU]).To(BeEquivalentTo("15.54%, 7600m/48900m")) + Expect(mp.Status.ReservedCapacity[v1.ResourceMemory]).To(BeEquivalentTo("20.45%, 77Gi/385500Mi")) + Expect(mp.Status.ReservedCapacity[v1.ResourcePods]).To(BeEquivalentTo("2.67%, 4/150")) ExpectDeleted(ns.Client, mp) ExpectDeleted(ns.Client, nodes...) diff --git a/pkg/controllers/provisioning/v1alpha1/allocator/bind.go b/pkg/controllers/provisioning/v1alpha1/allocator/bind.go new file mode 100644 index 000000000000..bf8f69a8e4fc --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/allocator/bind.go @@ -0,0 +1,90 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "context" + "fmt" + + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const KarpenterNodeLabelKey = "karpenter.sh/provisioner" + +type Binder struct { + kubeClient client.Client + coreV1Client *corev1.CoreV1Client +} + +func (b *Binder) Bind(ctx context.Context, provisioner *v1alpha1.Provisioner, node *v1.Node, pods []*v1.Pod) error { + // 1. Decorate node + b.decorate(provisioner, node) + + // 2. Create node + if err := b.create(ctx, node); err != nil { + return err + } + + // 3. Bind pods + for _, pod := range pods { + if err := b.bind(ctx, node, pod); err != nil { + zap.S().Errorf("Continuing after failing to bind, %s", err.Error()) + } else { + zap.S().Debugf("Successfully bound pod %s/%s to node %s", pod.Namespace, pod.Name, node.Name) + } + } + return nil +} + +func (b *Binder) decorate(provisioner *v1alpha1.Provisioner, node *v1.Node) { + node.ObjectMeta.Labels = map[string]string{ + KarpenterNodeLabelKey: fmt.Sprintf("%s-%s", provisioner.Name, provisioner.Namespace), + } + // Unfortunately, this detail is necessary to prevent kube-scheduler from + // scheduling pods to nodes before they're created. Node Lifecycle + // Controller will attach a Effect=NoSchedule taint in response to this + // condition and remove the taint when NodeReady=True. This behavior is + // stable, but may not be guaranteed to be true in the indefinite future. + // The failure mode in this case will unnecessarily create additional nodes. + // https://github.com/kubernetes/kubernetes/blob/f5fb1c93dbaa512eb66090c5027435d3dee95ac7/pkg/controller/nodelifecycle/node_lifecycle_controller.go#L86 + node.Status.Conditions = []v1.NodeCondition{{ + Type: v1.NodeReady, + Status: v1.ConditionUnknown, + }} +} + +func (a *Binder) create(ctx context.Context, node *v1.Node) error { + if _, err := a.coreV1Client.Nodes().Create(ctx, node, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("creating node %s, %w", node.Name, err) + } + return nil +} + +func (a *Binder) bind(ctx context.Context, node *v1.Node, pod *v1.Pod) error { + // TODO, Stop using deprecated v1.Binding + if err := a.coreV1Client.Pods(pod.Namespace).Bind(ctx, &v1.Binding{ + TypeMeta: pod.TypeMeta, + ObjectMeta: pod.ObjectMeta, + Target: v1.ObjectReference{Name: node.Name}, + }, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("binding pod, %w", err) + } + return nil +} diff --git a/pkg/controllers/provisioning/v1alpha1/allocator/constraints.go b/pkg/controllers/provisioning/v1alpha1/allocator/constraints.go new file mode 100644 index 000000000000..d80eb39d6214 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/allocator/constraints.go @@ -0,0 +1,64 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "github.com/awslabs/karpenter/pkg/cloudprovider" + v1 "k8s.io/api/core/v1" +) + +type Constraints struct{} + +func (c *Constraints) Group(pods []*v1.Pod) []*cloudprovider.Constraints { + groups := []*cloudprovider.Constraints{} + for _, pod := range pods { + added := false + for _, constraints := range groups { + if matchesConstraints(constraints, pod) { + constraints.Pods = append(constraints.Pods, pod) + added = true + break + } + } + if added { + continue + } + groups = append(groups, constraintsForPod(pod)) + } + return groups +} + +// TODO +func matchesConstraints(constraints *cloudprovider.Constraints, pod *v1.Pod) bool { + return false +} + +func constraintsForPod(pod *v1.Pod) *cloudprovider.Constraints { + return &cloudprovider.Constraints{ + Overhead: calculateOverheadResources(), + Architecture: getSystemArchitecture(pod), + Topology: map[cloudprovider.TopologyKey]string{}, + Pods: []*v1.Pod{pod}, + } +} + +func calculateOverheadResources() v1.ResourceList { + //TODO + return v1.ResourceList{} +} + +func getSystemArchitecture(pod *v1.Pod) cloudprovider.Architecture { + return cloudprovider.ArchitectureLinux386 +} diff --git a/pkg/controllers/provisioning/v1alpha1/allocator/controller.go b/pkg/controllers/provisioning/v1alpha1/allocator/controller.go new file mode 100644 index 000000000000..e34ab3b5e4b8 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/allocator/controller.go @@ -0,0 +1,103 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "context" + "fmt" + "time" + + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/cloudprovider" + "github.com/awslabs/karpenter/pkg/controllers" + "go.uber.org/zap" + corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Controller for the resource +type Controller struct { + filter *Filter + binder *Binder + constraints *Constraints + cloudProvider cloudprovider.Factory +} + +// For returns the resource this controller is for. +func (c *Controller) For() controllers.Object { + return &v1alpha1.Provisioner{} +} + +// Owns returns the resources owned by this controller's resource. +func (c *Controller) Owns() []controllers.Object { + return []controllers.Object{} +} + +func (c *Controller) Interval() time.Duration { + return 5 * time.Second +} + +func (c *Controller) Name() string { + return "provisioner/allocator" +} + +// NewController constructs a controller instance +func NewController(kubeClient client.Client, coreV1Client *corev1.CoreV1Client, cloudProvider cloudprovider.Factory) *Controller { + return &Controller{ + cloudProvider: cloudProvider, + filter: &Filter{kubeClient: kubeClient}, + binder: &Binder{kubeClient: kubeClient, coreV1Client: coreV1Client}, + constraints: &Constraints{}, + } +} + +// Reconcile executes an allocation control loop for the resource +func (c *Controller) Reconcile(object controllers.Object) error { + provisioner := object.(*v1alpha1.Provisioner) + ctx := context.TODO() + + // 1. Filter pods + pods, err := c.filter.GetProvisionablePods(ctx) + if err != nil { + return fmt.Errorf("filtering pods, %w", err) + } + if len(pods) == 0 { + return nil + } + zap.S().Infof("Found %d provisionable pods", len(pods)) + + // 2. Group by constraints + constraintGroups := c.constraints.Group(pods) + + // 3. Create capacity and packings + var packings []cloudprovider.Packing + for _, constraints := range constraintGroups { + packing, err := c.cloudProvider.CapacityFor(&provisioner.Spec).Create(ctx, constraints) + if err != nil { + zap.S().Errorf("Continuing after failing to create capacity, %w", err) + } else { + packings = append(packings, packing...) + } + } + + // 4. Bind pods to nodes + for _, packing := range packings { + zap.S().Infof("Binding %d pods to node %s", len(pods), packing.Node.Name) + if err := c.binder.Bind(ctx, provisioner, packing.Node, packing.Pods); err != nil { + zap.S().Errorf("Continuing after failing to bind, %w", err) + } + } + return nil +} diff --git a/pkg/controllers/provisioning/v1alpha1/allocator/filter.go b/pkg/controllers/provisioning/v1alpha1/allocator/filter.go new file mode 100644 index 000000000000..ef515d36c938 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/allocator/filter.go @@ -0,0 +1,46 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "context" + "fmt" + "github.com/awslabs/karpenter/pkg/controllers/provisioning/v1alpha1/util/scheduling" + + "github.com/awslabs/karpenter/pkg/utils/ptr" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Filter struct { + kubeClient client.Client +} + +func (f *Filter) GetProvisionablePods(ctx context.Context) ([]*v1.Pod, error) { + // 1. List Pods that aren't scheduled + pods := &v1.PodList{} + if err := f.kubeClient.List(ctx, pods, client.MatchingFields{"spec.nodeName": ""}); err != nil { + return nil, fmt.Errorf("listing unscheduled pods, %w", err) + } + + // 2. Filter pods that aren't provisionable + provisionable := []*v1.Pod{} + for _, pod := range pods.Items { + if scheduling.IsUnschedulable(&pod) && scheduling.IsNotIgnored(&pod) { + provisionable = append(provisionable, ptr.Pod(pod)) + } + } + return provisionable, nil +} diff --git a/pkg/controllers/provisioning/v1alpha1/allocator/suite_test.go b/pkg/controllers/provisioning/v1alpha1/allocator/suite_test.go new file mode 100644 index 000000000000..91c9db867cb0 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/allocator/suite_test.go @@ -0,0 +1,65 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "context" + "testing" + + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/test/environment" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, + "Provisioner", + []Reporter{printer.NewlineReporter{}}) +} + +var env environment.Environment = environment.NewLocal(func(e *environment.Local) { + e.Manager.Register(&Controller{}) +}) + +var _ = BeforeSuite(func() { + Expect(env.Start()).To(Succeed(), "Failed to start environment") +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = Describe("Provisioner", func() { + var ns *environment.Namespace + var p *v1alpha1.Provisioner + + BeforeEach(func() { + var err error + ns, err = env.NewNamespace() + Expect(err).NotTo(HaveOccurred()) + p = &v1alpha1.Provisioner{} + }) + + Context("Provisioner", func() { + It("should do something", func() { + Expect(ns.ParseResources("docs/examples/provisioner/provisioner.yaml", p)).To(Succeed()) + Expect(ns.Create(context.TODO(), p)).To(Succeed()) + }) + }) +}) diff --git a/pkg/controllers/provisioning/v1alpha1/reallocator/controller.go b/pkg/controllers/provisioning/v1alpha1/reallocator/controller.go new file mode 100644 index 000000000000..c2265ec27298 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/reallocator/controller.go @@ -0,0 +1,83 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reallocator + +import ( + "context" + "fmt" + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/cloudprovider" + "github.com/awslabs/karpenter/pkg/controllers" + "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +// Controller for the resource +type Controller struct { + filter *Filter + cloudProvider cloudprovider.Factory +} + +// For returns the resource this controller is for. +func (c *Controller) For() controllers.Object { + return &v1alpha1.Provisioner{} +} + +// Owns returns the resources owned by this controller's resource. +func (c *Controller) Owns() []controllers.Object { + return []controllers.Object{} +} + +func (c *Controller) Interval() time.Duration { + return 5 * time.Second +} + +func (c *Controller) Name() string { + return "provisioner/reallocator" +} + +// NewController constructs a controller instance +func NewController(kubeClient client.Client, cloudProvider cloudprovider.Factory) *Controller { + return &Controller{ + filter: &Filter{kubeClient: kubeClient}, + cloudProvider: cloudProvider, + } +} + +// Reconcile executes an allocation control loop for the resource +func (c *Controller) Reconcile(object controllers.Object) error { + ctx := context.TODO() + // 1. Filter all nodes with Karpenter TTL label + // TODO: Filter only nodes with the Provisioner label + + // 2. Remove TTL from nodes with TTL label that have pods and if past TTL, cordon/drain node + + // 3. Filter under-utilized nodes + nodes, err := c.filter.GetUnderutilizedNodes(ctx) + if err != nil { + return fmt.Errorf("filtering nodes, %w", err) + } + if len(nodes) == 0 { + return nil + } + zap.S().Infof("Found %d underutilized nodes", len(nodes)) + + // 4. Cordon each node + + // 5. Put TTL of 300s on each node + + return nil +} diff --git a/pkg/controllers/provisioning/v1alpha1/reallocator/filter.go b/pkg/controllers/provisioning/v1alpha1/reallocator/filter.go new file mode 100644 index 000000000000..05aa5552551d --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/reallocator/filter.go @@ -0,0 +1,64 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reallocator + +import ( + "context" + "fmt" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Filter struct { + kubeClient client.Client +} + +// Gets Nodes fitting some underutilization +// TODO: add predicate +func (f *Filter) GetUnderutilizedNodes(ctx context.Context) ([]*v1.Node, error) { + nodeList := &v1.NodeList{} + underutilized := []*v1.Node{} + + // 1. Get all nodes + if err := f.kubeClient.List(ctx, nodeList); err != nil { + return nil, fmt.Errorf("listing nodes, %w", err) + } + + // 2. Get nodes and the pods on each node + for _, node := range nodeList.Items { + pods, err := f.getPodsOnNode(ctx, node.Name) + if err != nil { + return []*v1.Node{}, fmt.Errorf("filtering pods on nodes, %w", err) + } + if f.isUnderutilized(&node, pods) { + underutilized = append(underutilized, &node) + } + } + return underutilized, nil +} + +// Get Pods scheduled to a node +func (f *Filter) getPodsOnNode(ctx context.Context, nodeName string) (*v1.PodList, error) { + pods := &v1.PodList{} + if err := f.kubeClient.List(ctx, pods, client.MatchingFields{"spec.nodeName": nodeName}); err != nil { + return nil, fmt.Errorf("listing unscheduled pods, %w", err) + } + return pods, nil +} + +// TODO: implement underutilized function +func (f *Filter) isUnderutilized(*v1.Node, *v1.PodList) bool { + return false +} diff --git a/pkg/controllers/provisioning/v1alpha1/reallocator/suite_test.go b/pkg/controllers/provisioning/v1alpha1/reallocator/suite_test.go new file mode 100644 index 000000000000..db38bc6b15f3 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/reallocator/suite_test.go @@ -0,0 +1,65 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package reallocator + +import ( + "context" + "testing" + + "github.com/awslabs/karpenter/pkg/apis/provisioning/v1alpha1" + "github.com/awslabs/karpenter/pkg/test/environment" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/envtest/printer" +) + +func TestAPIs(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecsWithDefaultAndCustomReporters(t, + "Provisioner", + []Reporter{printer.NewlineReporter{}}) +} + +var env environment.Environment = environment.NewLocal(func(e *environment.Local) { + e.Manager.Register(&Controller{}) +}) + +var _ = BeforeSuite(func() { + Expect(env.Start()).To(Succeed(), "Failed to start environment") +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = Describe("Provisioner", func() { + var ns *environment.Namespace + var p *v1alpha1.Provisioner + + BeforeEach(func() { + var err error + ns, err = env.NewNamespace() + Expect(err).NotTo(HaveOccurred()) + p = &v1alpha1.Provisioner{} + }) + + Context("Provisioner", func() { + It("should do something", func() { + Expect(ns.ParseResources("docs/examples/provisioner/provisioner.yaml", p)).To(Succeed()) + Expect(ns.Create(context.TODO(), p)).To(Succeed()) + }) + }) +}) diff --git a/pkg/controllers/provisioning/v1alpha1/util/scheduling/util.go b/pkg/controllers/provisioning/v1alpha1/util/scheduling/util.go new file mode 100644 index 000000000000..1860f57bf6d4 --- /dev/null +++ b/pkg/controllers/provisioning/v1alpha1/util/scheduling/util.go @@ -0,0 +1,48 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduling + +import ( + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + IgnoredOwners []schema.GroupVersionKind = []schema.GroupVersionKind{ + {Group: "apps", Version: "v1", Kind: "DaemonSet"}, + } +) + +func IsNotIgnored(pod *v1.Pod) bool { + for _, ignoredOwner := range IgnoredOwners { + for _, owner := range pod.ObjectMeta.OwnerReferences { + if owner.APIVersion == ignoredOwner.GroupVersion().String() && owner.Kind == ignoredOwner.Kind { + zap.S().Debugf("Ignoring %s %s %s/%s", owner.APIVersion, owner.Kind, pod.Namespace, owner.Name) + return false + } + } + } + return true +} + +func IsUnschedulable(pod *v1.Pod) bool { + for _, condition := range pod.Status.Conditions { + if condition.Type == v1.PodScheduled && condition.Reason == v1.PodReasonUnschedulable { + return true + } + } + return false +} diff --git a/pkg/controllers/scalablenodegroup/v1alpha1/controller.go b/pkg/controllers/scalablenodegroup/v1alpha1/controller.go index 6a16f85a69d3..f8ee54b54ae4 100644 --- a/pkg/controllers/scalablenodegroup/v1alpha1/controller.go +++ b/pkg/controllers/scalablenodegroup/v1alpha1/controller.go @@ -64,7 +64,7 @@ func (c *Controller) reconcile(resource *v1alpha1.ScalableNodeGroup) error { if err != nil { return fmt.Errorf("unable to get replica count for node group %v, %w", resource.Spec.ID, err) } - resource.Status.Replicas = observedReplicas + resource.Status.Replicas = &observedReplicas // Set desired replicas if different that current. if resource.Spec.Replicas == nil || *resource.Spec.Replicas == observedReplicas { diff --git a/pkg/controllers/scalablenodegroup/v1alpha1/suite_test.go b/pkg/controllers/scalablenodegroup/v1alpha1/suite_test.go index 15e4fe57e702..beb3a12a02e6 100644 --- a/pkg/controllers/scalablenodegroup/v1alpha1/suite_test.go +++ b/pkg/controllers/scalablenodegroup/v1alpha1/suite_test.go @@ -15,6 +15,7 @@ limitations under the License. package v1alpha1 import ( + "fmt" "testing" v1alpha1 "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" @@ -38,9 +39,9 @@ func TestAPIs(t *testing.T) { } var fakeCloudProvider = fake.NewFactory(cloudprovider.Options{}) - +var fakeController = &Controller{CloudProvider: fakeCloudProvider} var env environment.Environment = environment.NewLocal(func(e *environment.Local) { - e.Manager.Register(&Controller{CloudProvider: fakeCloudProvider}) + e.Manager.Register(fakeController) }) var _ = BeforeSuite(func() { @@ -53,6 +54,8 @@ var _ = AfterSuite(func() { var _ = Describe("Examples", func() { var ns *environment.Namespace + var defaultReplicaCount = int32(3) + var sng *v1alpha1.ScalableNodeGroup BeforeEach(func() { @@ -60,17 +63,63 @@ var _ = Describe("Examples", func() { ns, err = env.NewNamespace() Expect(err).NotTo(HaveOccurred()) sng = &v1alpha1.ScalableNodeGroup{} + fakeCloudProvider.NodeGroupStable = true + fakeCloudProvider.WantErr = nil + sng.Spec.Replicas = &defaultReplicaCount + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = ptr.Int32(0) }) Context("ScalableNodeGroup", func() { It("should be created", func() { Expect(ns.ParseResources("docs/examples/reserved-capacity-utilization.yaml", sng)).To(Succeed()) - sng.Spec.Replicas = ptr.Int32(5) - + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = ptr.Int32(*sng.Spec.Replicas) ExpectCreated(ns.Client, sng) ExpectEventuallyHappy(ns.Client, sng) - ExpectDeleted(ns.Client, sng) }) }) + + Context("ScalableNodeGroup Reconcile tests", func() { + It("Test reconciler to scale up nodes", func() { + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(*fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(defaultReplicaCount)) + }) + + It("Test reconciler to scale down nodes", func() { + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = ptr.Int32(10) // set existing replicas higher than desired + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(*fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(defaultReplicaCount)) + }) + + It("Test reconciler to make no change to node count", func() { + fakeCloudProvider.NodeReplicas[sng.Spec.ID] = &defaultReplicaCount // set existing replicas equal to desired + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(*fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(defaultReplicaCount)) + }) + + It("Scale up nodes when not node group is stabilized and check status condition", func() { + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(*fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(defaultReplicaCount)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.Stabilized).IsTrue()).To(Equal(true)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.Stabilized).Message).To(Equal("")) + }) + + It("Scale up nodes when not node group is NOT stabilized and check status condition", func() { + fakeCloudProvider.NodeGroupStable = false + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(*fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(defaultReplicaCount)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.Stabilized).IsFalse()).To(Equal(true)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.Stabilized).Message).To(Equal(fake.NodeGroupMessage)) + }) + + It("Retryable error while reconciling", func() { + fakeCloudProvider.WantErr = fake.RetryableError(fmt.Errorf(fake.NodeGroupMessage)) // retryable error + existingReplicas := fakeCloudProvider.NodeReplicas[sng.Spec.ID] + Expect(fakeController.Reconcile(sng)).To(Succeed()) + Expect(fakeCloudProvider.NodeReplicas[sng.Spec.ID]).To(Equal(existingReplicas)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.AbleToScale).IsFalse()).To(Equal(true)) + Expect(sng.StatusConditions().GetCondition(v1alpha1.AbleToScale).Message).To(Equal(fake.NodeGroupMessage)) + }) + + }) }) diff --git a/pkg/metrics/producers/factory.go b/pkg/metrics/producers/factory.go index 254623a52d63..85e8ffe8babc 100644 --- a/pkg/metrics/producers/factory.go +++ b/pkg/metrics/producers/factory.go @@ -34,9 +34,6 @@ type Factory struct { } func (f *Factory) For(mp *v1alpha1.MetricsProducer) metrics.Producer { - if err := mp.Spec.Validate(); err != nil { - return &fake.FakeProducer{WantErr: err} - } if mp.Spec.PendingCapacity != nil { return &pendingcapacity.Producer{ MetricsProducer: mp, @@ -55,7 +52,7 @@ func (f *Factory) For(mp *v1alpha1.MetricsProducer) metrics.Producer { Client: f.Client, } } - if mp.Spec.ScheduledCapacity != nil { + if mp.Spec.Schedule != nil { return &scheduledcapacity.Producer{ MetricsProducer: mp, } diff --git a/pkg/metrics/producers/reservedcapacity/gauges.go b/pkg/metrics/producers/reservedcapacity/gauges.go index 843e4654a0d2..c8532b4feae4 100644 --- a/pkg/metrics/producers/reservedcapacity/gauges.go +++ b/pkg/metrics/producers/reservedcapacity/gauges.go @@ -16,21 +16,29 @@ package reservedcapacity import ( "fmt" + "github.com/awslabs/karpenter/pkg/metrics" + "github.com/prometheus/client_golang/prometheus" v1 "k8s.io/api/core/v1" ) +type MetricType string + const ( Subsystem = "reserved_capacity" - Reserved = "reserved" - Capacity = "capacity" - Utilization = "utilization" + Reserved = MetricType("reserved") + Capacity = MetricType("capacity") + Utilization = MetricType("utilization") ) func init() { - for _, resource := range []v1.ResourceName{v1.ResourcePods, v1.ResourceCPU, v1.ResourceMemory} { - for _, name := range []string{Reserved, Capacity, Utilization} { - metrics.RegisterNewGauge(Subsystem, fmt.Sprintf("%s_%s", resource, name)) + for _, resourceName := range []v1.ResourceName{v1.ResourcePods, v1.ResourceCPU, v1.ResourceMemory} { + for _, metricType := range []MetricType{Reserved, Capacity, Utilization} { + metrics.RegisterNewGauge(Subsystem, fmt.Sprintf("%s_%s", resourceName, metricType)) } } } + +func GaugeFor(resourceName v1.ResourceName, metricType MetricType) *prometheus.GaugeVec { + return metrics.Gauges[Subsystem][fmt.Sprintf("%s_%s", resourceName, metricType)] +} diff --git a/pkg/metrics/producers/reservedcapacity/producer.go b/pkg/metrics/producers/reservedcapacity/producer.go index 617f66e93dc6..63eaae6d121d 100644 --- a/pkg/metrics/producers/reservedcapacity/producer.go +++ b/pkg/metrics/producers/reservedcapacity/producer.go @@ -18,8 +18,7 @@ import ( "context" "fmt" "math" - - "github.com/awslabs/karpenter/pkg/metrics" + "strconv" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" "github.com/awslabs/karpenter/pkg/utils/node" @@ -66,16 +65,22 @@ func (p *Producer) record(reservations *Reservations) { p.Status.ReservedCapacity = map[v1.ResourceName]string{} } for resource, reservation := range reservations.Resources { - computed := reservation.Compute() - for metricType, value := range computed { - metrics.Gauges[Subsystem][fmt.Sprintf("%s_%s", resource, metricType)]. - WithLabelValues(p.Name, p.Namespace).Set(value) + reserved, _ := strconv.ParseFloat(reservation.Reserved.AsDec().String(), 64) + capacity, _ := strconv.ParseFloat(reservation.Capacity.AsDec().String(), 64) + utilization := math.NaN() + if capacity != 0 { + utilization = reserved / capacity } + + GaugeFor(resource, Utilization).WithLabelValues(p.Name, p.Namespace).Set(utilization) + GaugeFor(resource, Reserved).WithLabelValues(p.Name, p.Namespace).Set(reserved) + GaugeFor(resource, Capacity).WithLabelValues(p.Name, p.Namespace).Set(capacity) + p.Status.ReservedCapacity[resource] = fmt.Sprintf( - "%v%%, %d/%d", - math.Floor(computed[Utilization]*100), - int64(computed[Reserved]), - int64(computed[Capacity]), + "%.2f%%, %v/%v", + reserved/capacity*100, + reservation.Reserved, + reservation.Capacity, ) } } diff --git a/pkg/metrics/producers/reservedcapacity/reservations.go b/pkg/metrics/producers/reservedcapacity/reservations.go index 8f418fe8f283..99533434db3c 100644 --- a/pkg/metrics/producers/reservedcapacity/reservations.go +++ b/pkg/metrics/producers/reservedcapacity/reservations.go @@ -15,9 +15,6 @@ limitations under the License. package reservedcapacity import ( - "math" - "math/big" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" ) @@ -53,25 +50,12 @@ func (r *Reservations) Add(node *v1.Node, pods *v1.PodList) { r.Resources[v1.ResourceMemory].Reserved.Add(*container.Resources.Requests.Memory()) } } - r.Resources[v1.ResourcePods].Capacity.Add(*node.Status.Capacity.Pods()) - r.Resources[v1.ResourceCPU].Capacity.Add(*node.Status.Capacity.Cpu()) - r.Resources[v1.ResourceMemory].Capacity.Add(*node.Status.Capacity.Memory()) + r.Resources[v1.ResourcePods].Capacity.Add(*node.Status.Allocatable.Pods()) + r.Resources[v1.ResourceCPU].Capacity.Add(*node.Status.Allocatable.Cpu()) + r.Resources[v1.ResourceMemory].Capacity.Add(*node.Status.Allocatable.Memory()) } type Reservation struct { Reserved *resource.Quantity Capacity *resource.Quantity } - -func (r *Reservation) Compute() map[string]float64 { - var utilization = math.NaN() - if r.Capacity.Value() != 0 { - utilization, _ = big.NewRat(r.Reserved.Value(), r.Capacity.Value()).Float64() - } - - return map[string]float64{ - Reserved: float64(r.Reserved.Value()), - Capacity: float64(r.Capacity.Value()), - Utilization: utilization, - } -} diff --git a/pkg/metrics/producers/scheduledcapacity/crontabs.go b/pkg/metrics/producers/scheduledcapacity/crontabs.go new file mode 100644 index 000000000000..c76692ee0c35 --- /dev/null +++ b/pkg/metrics/producers/scheduledcapacity/crontabs.go @@ -0,0 +1,73 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduledcapacity + +import ( + "fmt" + "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + "github.com/robfig/cron/v3" + "log" + "strconv" + "strings" + "time" +) + +type Crontab struct { + string + location *time.Location +} + +// crontabFrom returns a Crontab equivalent to the strongly-typed schedule +func crontabFrom(pattern *v1alpha1.Pattern, location *time.Location) *Crontab { + return &Crontab{ + string: fmt.Sprintf("%s %s %s %s %s", crontabFieldFrom(pattern.Minutes, "0"), + crontabFieldFrom(pattern.Hours, "0"), crontabFieldFrom(pattern.Days, "*"), + crontabFieldFrom(pattern.Months, "*"), crontabFieldFrom(pattern.Weekdays, "*")), + location: location, + } +} + +// crontabFieldFrom returns a field of a strongly-typed format into a Crontab field +func crontabFieldFrom(field *string, nilDefault string) string { + if field == nil { + return nilDefault + } + elements := strings.Split(*field, ",") + for i, val := range elements { + if _, err := strconv.Atoi(val); err != nil { + elements[i] = strings.ToUpper(val) + } else { + elements[i] = val + } + elements[i] = strings.Trim(val, " ") + } + return strings.Join(elements, ",") +} + +// nextTime returns the next time that Crontab will match in its timezone location +func (tab *Crontab) nextTime() (time.Time, error) { + c := cron.New(cron.WithLocation(tab.location)) + // AddFunc parses the Crontab into a job for the object to use below + _, err := c.AddFunc(tab.string, func() { log.Printf("crontab %s has been initialized", tab.string) }) + if err != nil { + return time.Time{}, fmt.Errorf("could not parse crontab: %w", err) + } + + c.Start() + defer c.Stop() + nextTime := c.Entries()[0].Next + + return nextTime, nil +} diff --git a/pkg/metrics/producers/scheduledcapacity/gauges.go b/pkg/metrics/producers/scheduledcapacity/gauges.go new file mode 100644 index 000000000000..be8314a67ca9 --- /dev/null +++ b/pkg/metrics/producers/scheduledcapacity/gauges.go @@ -0,0 +1,26 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheduledcapacity + +import "github.com/awslabs/karpenter/pkg/metrics" + +const ( + Subsystem = "scheduled_replicas" + Value = "value" +) + +func init() { + metrics.RegisterNewGauge(Subsystem, Value) +} diff --git a/pkg/metrics/producers/scheduledcapacity/producer.go b/pkg/metrics/producers/scheduledcapacity/producer.go index 7aa0f9d9f4cc..05242f545c27 100644 --- a/pkg/metrics/producers/scheduledcapacity/producer.go +++ b/pkg/metrics/producers/scheduledcapacity/producer.go @@ -15,7 +15,10 @@ limitations under the License. package scheduledcapacity import ( + "fmt" "github.com/awslabs/karpenter/pkg/apis/autoscaling/v1alpha1" + "github.com/awslabs/karpenter/pkg/metrics" + "time" ) // Producer implements the ScheduledCapacity metric @@ -25,5 +28,44 @@ type Producer struct { // Reconcile of the metrics func (p *Producer) Reconcile() error { + var ( + loc *time.Location + err error + ) + // defaulting webhook ensures this is always defined + loc, err = time.LoadLocation(*p.Spec.Schedule.Timezone) + if err != nil { + return fmt.Errorf("timezone was not a valid input") + } + now := time.Now().In(loc) + + p.Status.ScheduledCapacity = &v1alpha1.ScheduledCapacityStatus{ + CurrentValue: &p.Spec.Schedule.DefaultReplicas, + } + + for _, behavior := range p.Spec.Schedule.Behaviors { + // use Cron library to find the next time start and end next match + startTime, err := crontabFrom(behavior.Start, loc).nextTime() + if err != nil { + return fmt.Errorf("start pattern is invalid: %w", err) + } + endTime, err := crontabFrom(behavior.End, loc).nextTime() + if err != nil { + return fmt.Errorf("end pattern is invalid: %w", err) + } + + if !now.After(endTime) && (!endTime.After(startTime) || !startTime.After(now)) { + // Since the way collisions are handled are by how they're ordered in the spec, stop on first match + p.Status.ScheduledCapacity.CurrentValue = &behavior.Replicas + break + } + } + p.record() return nil } + +func (p *Producer) record() { + metrics.Gauges[Subsystem][Value]. + WithLabelValues(p.Name, p.Namespace). + Set(float64(*p.Status.ScheduledCapacity.CurrentValue)) +} diff --git a/pkg/test/objects.go b/pkg/test/objects.go index 7da209358cd0..706214aa1920 100644 --- a/pkg/test/objects.go +++ b/pkg/test/objects.go @@ -46,7 +46,7 @@ type NodeOptions struct { Labels map[string]string ReadyStatus v1.ConditionStatus Unschedulable bool - Capacity v1.ResourceList + Allocatable v1.ResourceList } func NodeWith(options NodeOptions) *v1.Node { @@ -69,8 +69,8 @@ func NodeWith(options NodeOptions) *v1.Node { Unschedulable: options.Unschedulable, }, Status: v1.NodeStatus{ - Capacity: options.Capacity, - Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: options.ReadyStatus}}, + Allocatable: options.Allocatable, + Conditions: []v1.NodeCondition{{Type: v1.NodeReady, Status: options.ReadyStatus}}, }, } } diff --git a/pkg/utils/ptr/ptr.go b/pkg/utils/ptr/ptr.go new file mode 100644 index 000000000000..8be31a439ad2 --- /dev/null +++ b/pkg/utils/ptr/ptr.go @@ -0,0 +1,21 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ptr + +import v1 "k8s.io/api/core/v1" + +func Pod(pod v1.Pod) *v1.Pod { + return &pod +} diff --git a/releases/aws/manifest.yaml b/releases/aws/manifest.yaml new file mode 100644 index 000000000000..c94d3b52d406 --- /dev/null +++ b/releases/aws/manifest.yaml @@ -0,0 +1,1226 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: karpenter + name: karpenter +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: karpenter/karpenter-serving-cert + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + labels: + control-plane: karpenter + name: horizontalautoscalers.autoscaling.karpenter.sh +spec: + group: autoscaling.karpenter.sh + names: + kind: HorizontalAutoscaler + listKind: HorizontalAutoscalerList + plural: horizontalautoscalers + shortNames: + - horizontalautoscaler + singular: horizontalautoscaler + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.minReplicas + name: min + type: string + - jsonPath: .status.desiredReplicas + name: desired + type: string + - jsonPath: .spec.maxReplicas + name: max + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: ready + type: string + - jsonPath: .status.lastScaleTime + name: last-scale-time + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: HorizontalAutoscaler is the Schema for the horizontalautoscalers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HorizontalAutoscalerSpec is modeled after https://godoc.org/k8s.io/api/autoscaling/v2beta2#HorizontalPodAutoscalerSpec This enables parity of functionality between Pod and Node autoscaling, with a few minor differences. 1. ObjectSelector is replaced by NodeSelector. 2. Metrics.PodsMetricSelector is replaced by the more generic Metrics.ReplicaMetricSelector. + properties: + behavior: + description: Behavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). If not set, the default ScalingRules for scale up and scale down are used. + properties: + scaleDown: + description: ScaleDown is scaling policy for scaling Down. If not set, the default value is to allow to scale down to minReplicas, with a 300 second stabilization window (i.e., the highest recommendation for the last 300sec is used). + properties: + policies: + description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the ScalingRules will be discarded as invalid + items: + description: ScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling policy. + type: string + value: + description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + format: int32 + type: integer + type: object + scaleUp: + description: 'ScaleUp is scaling policy for scaling Up. If not set, the default value is the higher of: * increase no more than 4 replicas per 60 seconds * double the number of replicas per 60 seconds No stabilization is used.' + properties: + policies: + description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the ScalingRules will be discarded as invalid + items: + description: ScalingPolicy is a single policy which must hold true for a specified past interval. + properties: + periodSeconds: + description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + format: int32 + type: integer + type: + description: Type is used to specify the scaling policy. + type: string + value: + description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + type: string + stabilizationWindowSeconds: + description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + format: int32 + type: integer + type: object + type: object + maxReplicas: + description: MaxReplicas is the upper limit for the number of replicas to which the autoscaler can scale up. It cannot be less that minReplicas. + format: int32 + type: integer + metrics: + description: Metrics contains the specifications for which to use to calculate the desired replica count (the maximum replica count across all metrics will be used). The desired replica count is calculated multiplying the ratio between the target value and the current value by the current number of replicas. Ergo, metrics used must decrease as the replica count is increased, and vice-versa. See the individual metric source types for more information about how each type of metric must respond. If not set, the default metric will be set to 80% average CPU utilization. + items: + description: Metric is modeled after https://godoc.org/k8s.io/api/autoscaling/v2beta2#MetricSpec + properties: + prometheus: + description: PrometheusMetricSource defines a metric in Prometheus + properties: + query: + type: string + target: + description: MetricTarget defines the target value, average value, or average utilization of a specific metric + properties: + averageUtilization: + description: AverageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. Currently only valid for Resource metric source type + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: AverageValue is the target value of the average of the metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + description: Type represents whether the metric type is Utilization, Value, or AverageValue Value is the target value of the metric (as a quantity). + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - query + - target + type: object + type: object + type: array + minReplicas: + description: MinReplicas is the lower limit for the number of replicas to which the autoscaler can scale down. It is allowed to be 0. + format: int32 + type: integer + scaleTargetRef: + description: ScaleTargetRef points to the target resource to scale. + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - minReplicas + - scaleTargetRef + type: object + status: + description: HorizontalAutoscalerStatus defines the observed state of HorizontalAutoscaler + properties: + conditions: + description: Conditions is the set of conditions required for this autoscaler to scale its target, and indicates whether or not those conditions are met. + items: + description: 'Conditions defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + currentMetrics: + description: CurrentMetrics is the last read state of the metrics used by this autoscaler. + items: + description: MetricStatus contains status information for the configured metrics source. This status has a one-of semantic and will only ever contain one value. + properties: + prometheus: + properties: + current: + description: Current contains the current value for the given metric + properties: + averageUtilization: + description: currentAverageUtilization is the current value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + description: AverageValue is the current value of the average of the metric across all relevant pods (as a quantity) + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + value: + anyOf: + - type: integer + - type: string + description: Value is the current value of the metric (as a quantity). + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + query: + description: Query of the metric + type: string + required: + - current + - query + type: object + type: object + type: array + currentReplicas: + description: CurrentReplicas is current number of replicas of pods managed by this autoscaler, as last seen by the autoscaler. + format: int32 + type: integer + desiredReplicas: + description: DesiredReplicas is the desired number of replicas of pods managed by this autoscaler, as last calculated by the autoscaler. + format: int32 + type: integer + lastScaleTime: + description: LastScaleTime is the last time the HorizontalAutoscaler scaled the number of pods, used by the autoscaler to control how often the number of pods is changed. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + labels: + control-plane: karpenter + name: metricsproducers.autoscaling.karpenter.sh +spec: + group: autoscaling.karpenter.sh + names: + kind: MetricsProducer + listKind: MetricsProducerList + plural: metricsproducers + shortNames: + - metricsproducer + singular: metricsproducer + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: ready + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: MetricsProducer is the Schema for the MetricsProducers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: MetricsProducerSpec defines an object that outputs metrics. + properties: + pendingCapacity: + description: PendingCapacity produces a metric that recommends increases or decreases to the sizes of a set of node groups based on pending pods. + properties: + nodeSelector: + additionalProperties: + type: string + description: NodeSelector specifies a node group. The selector must uniquely identify a set of nodes. + type: object + required: + - nodeSelector + type: object + queue: + description: Queue produces metrics about a specified queue, such as length and age of oldest message, + properties: + id: + type: string + type: + description: QueueType corresponds to an implementation of a queue + type: string + required: + - id + - type + type: object + reservedCapacity: + description: ReservedCapacity produces a metric corresponding to the ratio of committed resources to available resources for the nodes of a specified node group. + properties: + nodeSelector: + additionalProperties: + type: string + description: NodeSelector specifies a node group. The selector must uniquely identify a set of nodes. + type: object + required: + - nodeSelector + type: object + scheduleSpec: + description: Schedule produces a metric according to a specified schedule. + properties: + behaviors: + description: Behaviors may be layered to achieve complex scheduling autoscaling logic + items: + description: ScheduledBehavior sets the metric to a replica value based on a start and end pattern. + properties: + end: + description: Pattern is a strongly-typed version of crontabs + properties: + days: + description: When Days, Months, or Weekdays are left out, they are represented by wildcards, meaning any time matches + type: string + hours: + type: string + minutes: + description: When minutes or hours are left out, they are assumed to match to 0 + type: string + months: + description: List of 3-letter abbreviations i.e. Jan, Feb, Mar + type: string + weekdays: + description: List of 3-letter abbreviations i.e. "Mon, Tue, Wed" + type: string + type: object + replicas: + description: The value the MetricsProducer will emit when the current time is within start and end + format: int32 + type: integer + start: + description: Pattern is a strongly-typed version of crontabs + properties: + days: + description: When Days, Months, or Weekdays are left out, they are represented by wildcards, meaning any time matches + type: string + hours: + type: string + minutes: + description: When minutes or hours are left out, they are assumed to match to 0 + type: string + months: + description: List of 3-letter abbreviations i.e. Jan, Feb, Mar + type: string + weekdays: + description: List of 3-letter abbreviations i.e. "Mon, Tue, Wed" + type: string + type: object + required: + - end + - replicas + - start + type: object + type: array + defaultReplicas: + description: A schedule defaults to this value when no behaviors are active + format: int32 + type: integer + timezone: + description: 'Defaults to UTC. Users will specify their schedules assuming this is their timezone ref: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones' + type: string + required: + - behaviors + - defaultReplicas + type: object + type: object + status: + description: MetricsProducerStatus defines the observed state of the resource. + properties: + conditions: + description: Conditions is the set of conditions required for the metrics producer to successfully publish metrics to the metrics server + items: + description: 'Conditions defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + pendingCapacity: + type: object + queue: + properties: + length: + description: Length of the Queue + format: int64 + type: integer + oldestMessageAgeSeconds: + description: The age of the oldest message in the queue in seconds + format: int64 + type: integer + required: + - length + type: object + reservedCapacity: + additionalProperties: + type: string + type: object + scheduledCapacity: + properties: + currentValue: + description: The current recommendation - the metric the MetricsProducer is emitting + format: int32 + type: integer + nextValue: + description: Not Currently Implemented The next recommendation for the metric + format: int32 + type: integer + nextValueTime: + description: Not Currently Implemented The time in the future where CurrentValue will switch to NextValue + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + labels: + control-plane: karpenter + name: provisioners.provisioning.karpenter.sh +spec: + group: provisioning.karpenter.sh + names: + kind: Provisioner + listKind: ProvisionerList + plural: provisioners + singular: provisioner + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Provisioner is the Schema for the Provisioners API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + allocator: + description: AllocatorSpec configures node allocation policy + properties: + instanceTypes: + items: + type: string + type: array + type: object + cluster: + description: ClusterSpec configures the cluster that the provisioner operates against. If not specified, it will default to using the controller's kube-config. + properties: + caBundle: + description: CABundle is required for nodes to verify API Server certificates. + type: string + endpoint: + description: Endpoint is required for nodes to connect to the API Server. + type: string + name: + description: Name is required to detect implementing cloud provider resources. + type: string + required: + - caBundle + - endpoint + - name + type: object + deallocator: + description: DeallocatorSpec configures + type: object + type: object + status: + description: ProvisionerStatus defines the observed state of Provisioner + properties: + conditions: + description: Conditions is the set of conditions required for this provisioner to scale its target, and indicates whether or not those conditions are met. + items: + description: 'Conditions defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + lastScaleTime: + description: LastScaleTime is the last time the Provisioner scaled the number of nodes + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: karpenter/karpenter-serving-cert + controller-gen.kubebuilder.io/version: v0.4.0 + creationTimestamp: null + labels: + control-plane: karpenter + name: scalablenodegroups.autoscaling.karpenter.sh +spec: + group: autoscaling.karpenter.sh + names: + kind: ScalableNodeGroup + listKind: ScalableNodeGroupList + plural: scalablenodegroups + shortNames: + - scalablenodegroup + singular: scalablenodegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: desired + type: string + - jsonPath: .status.replicas + name: current + type: string + - jsonPath: .spec.type + name: type + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: ready + type: string + - jsonPath: .metadata.creationTimestamp + name: age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: ScalableNodeGroup is the Schema for the ScalableNodeGroups API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ScalableNodeGroupSpec is an abstract representation for a Cloud Provider's Node Group. It implements https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#scale-subresource which enables it to be targeted by Horizontal Pod Autoscalers. + properties: + id: + description: ID to identify the underlying resource + type: string + replicas: + description: Replicas is the desired number of replicas for the targeted Node Group + format: int32 + type: integer + type: + description: Type for the resource of name ScalableNodeGroup.ObjectMeta.Name + type: string + required: + - id + - type + type: object + status: + description: ScalableNodeGroupStatus holds status information for the ScalableNodeGroup + properties: + conditions: + description: Conditions is the set of conditions required for the scalable node group to successfully enforce the replica count of the underlying group + items: + description: 'Conditions defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + replicas: + description: Replicas displays the actual size of the ScalableNodeGroup at the time of the last reconciliation + format: int32 + type: integer + type: object + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: karpenter/karpenter-serving-cert + creationTimestamp: null + labels: + control-plane: karpenter + name: karpenter-mutating-webhook-configuration + namespace: karpenter +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /mutate-autoscaling-karpenter-sh-v1alpha1-horizontalautoscaler + failurePolicy: Fail + name: mhorizontalautoscaler.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - horizontalautoscalers + sideEffects: None + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /mutate-autoscaling-karpenter-sh-v1alpha1-metricsproducer + failurePolicy: Fail + name: mmetricsproducer.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - metricsproducers + sideEffects: None + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /mutate-autoscaling-karpenter-sh-v1alpha1-scalablenodegroup + failurePolicy: Fail + name: mscalablenodegroup.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - scalablenodegroups + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + cert-manager.io/inject-ca-from: karpenter/karpenter-serving-cert + creationTimestamp: null + labels: + control-plane: karpenter + name: karpenter-validating-webhook-configuration + namespace: karpenter +webhooks: + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /validate-autoscaling-karpenter-sh-v1alpha1-horizontalautoscaler + failurePolicy: Fail + name: vhorizontalautoscaler.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - horizontalautoscalers + sideEffects: None + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /validate-autoscaling-karpenter-sh-v1alpha1-metricsproducer + failurePolicy: Fail + name: vmetricsproducer.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - metricsproducers + sideEffects: None + - admissionReviewVersions: + - v1beta1 + clientConfig: + caBundle: Cg== + service: + name: karpenter-webhook-service + namespace: karpenter + path: /validate-autoscaling-karpenter-sh-v1alpha1-scalablenodegroup + failurePolicy: Fail + name: scalablenodegroup.kb.io + rules: + - apiGroups: + - autoscaling.karpenter.sh + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - scalablenodegroups + sideEffects: None +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + control-plane: karpenter + name: karpenter + namespace: karpenter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + control-plane: karpenter + name: karpenter-leader-election + namespace: karpenter +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - configmaps/status + verbs: + - get + - update + - patch + - apiGroups: + - "" + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + control-plane: karpenter + name: karpenter +rules: + - apiGroups: + - autoscaling.karpenter.sh + resources: + - horizontalautoscalers + - horizontalautoscalers/status + - metricsproducers + - metricsproducers/status + - scalablenodegroups + - scalablenodegroups/status + - provisioners + - provisioners/status + verbs: + - create + - delete + - patch + - get + - list + - patch + - watch + - apiGroups: + - provisioning.karpenter.sh + resources: + - provisioners + - provisioners/status + verbs: + - create + - delete + - patch + - get + - list + - patch + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - patch + - update + - watch + - apiGroups: + - '*' + resources: + - '*/scale' + verbs: + - update + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - create + - apiGroups: + - "" + resources: + - pods/binding + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + control-plane: karpenter + name: karpenter-metrics-reader +rules: + - nonResourceURLs: + - /metrics + verbs: + - get + - apiGroups: + - "" + resources: + - services + - endpoints + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + control-plane: karpenter + name: karpenter-leader-election + namespace: karpenter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: karpenter-leader-election +subjects: + - kind: ServiceAccount + name: karpenter + namespace: karpenter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + control-plane: karpenter + name: karpenter-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter-metrics-reader +subjects: + - kind: ServiceAccount + name: default + namespace: karpenter +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + control-plane: karpenter + name: karpenter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: karpenter +subjects: + - kind: ServiceAccount + name: karpenter + namespace: karpenter +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: karpenter + name: karpenter-webhook-service + namespace: karpenter +spec: + ports: + - port: 443 + targetPort: webhook + selector: + control-plane: karpenter +--- +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: karpenter + name: karpenter-metrics-service + namespace: karpenter +spec: + ports: + - name: http + port: 8080 + targetPort: metrics + selector: + control-plane: karpenter +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + control-plane: karpenter + name: karpenter + namespace: karpenter +spec: + replicas: 1 + selector: + matchLabels: + control-plane: karpenter + template: + metadata: + labels: + control-plane: karpenter + spec: + containers: + - image: public.ecr.aws/b6u6q9h4/controller:v0.1.2@sha256:42a628bfce91efb82007ade793a8ea6a5f69f345057e5c233eee8a05576930e1 + name: manager + ports: + - containerPort: 9443 + name: webhook + protocol: TCP + - containerPort: 8080 + name: metrics + resources: + limits: + cpu: 100m + memory: 30Mi + requests: + cpu: 100m + memory: 20Mi + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + securityContext: + fsGroup: 1000 + serviceAccountName: karpenter + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + control-plane: karpenter + name: karpenter-serving-cert + namespace: karpenter +spec: + dnsNames: + - karpenter-webhook-service.karpenter.svc + - karpenter-webhook-service.karpenter.svc.cluster.local + issuerRef: + kind: Issuer + name: karpenter-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + control-plane: karpenter + name: karpenter-selfsigned-issuer + namespace: karpenter +spec: + selfSigned: {} +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + labels: + control-plane: karpenter + name: karpenter-monitor + namespace: karpenter +spec: + replicas: 1 + serviceMonitorSelector: + matchLabels: + control-plane: karpenter +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: karpenter + name: karpenter-metrics-monitor + namespace: karpenter +spec: + endpoints: + - interval: 5s + path: /metrics + port: http + scheme: http + selector: + matchLabels: + control-plane: karpenter + +---