From 4a9ee095d1d1346d902467f003ac1ec87ff71536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lukas=20K=C3=A4mmerling?= Date: Thu, 21 Jan 2021 15:13:05 +0100 Subject: [PATCH] Rebase and change code owners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lukas Kämmerling --- .../cloudprovider/builder/builder_all.go | 3 +- .../cloudprovider/hetzner/OWNERS | 4 - .../cloudprovider/hetzner/README.md | 7 +- .../hetzner/examples/cloud-init.txt | 35 ++++ .../cluster-autoscaler-run-on-master.yaml | 190 ++++++++++++++++++ .../cloudprovider/hetzner/hetzner_manager.go | 29 ++- .../hetzner/hetzner_node_group.go | 15 +- hack/boilerplate/boilerplate.py | 3 +- hack/verify-gofmt.sh | 1 + hack/verify-golint.sh | 1 + 10 files changed, 274 insertions(+), 14 deletions(-) delete mode 100644 cluster-autoscaler/cloudprovider/hetzner/OWNERS create mode 100644 cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt create mode 100644 cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml diff --git a/cluster-autoscaler/cloudprovider/builder/builder_all.go b/cluster-autoscaler/cloudprovider/builder/builder_all.go index 00770e744a4c..e4ce5c2b18bf 100644 --- a/cluster-autoscaler/cloudprovider/builder/builder_all.go +++ b/cluster-autoscaler/cloudprovider/builder/builder_all.go @@ -10,7 +10,7 @@ You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS,! +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. @@ -29,6 +29,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/digitalocean" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/exoscale" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/gce" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/huaweicloud" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/ionoscloud" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/linode" diff --git a/cluster-autoscaler/cloudprovider/hetzner/OWNERS b/cluster-autoscaler/cloudprovider/hetzner/OWNERS deleted file mode 100644 index fb32465f61a2..000000000000 --- a/cluster-autoscaler/cloudprovider/hetzner/OWNERS +++ /dev/null @@ -1,4 +0,0 @@ -approvers: -- Fgruntjes -reviewers: -- Fgruntjes diff --git a/cluster-autoscaler/cloudprovider/hetzner/README.md b/cluster-autoscaler/cloudprovider/hetzner/README.md index 1a6758f06b9b..b3ba63c5ea8c 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/README.md +++ b/cluster-autoscaler/cloudprovider/hetzner/README.md @@ -5,9 +5,10 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes. # Configuration `HCLOUD_TOKEN` Required Hetzner Cloud token. -`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster +`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster, Sample [examples/cloud-init.txt for (Kubernetes 1.20.1)](examples/cloud-init.txt) `HCLOUD_IMAGE` Defaults to `ubuntu-20.04`, @see https://docs.hetzner.cloud/#images - +`HCLOUD_NETWORK` Default empty , The name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks +`HCLOUD_SSH_KEY` Default empty , This SSH Key will have access to the fresh created server, @see https://docs.hetzner.cloud/#ssh-keys Node groups must be defined with the `--nodes=::::` flag. Multiple flags will create multiple node pools. For example: ``` @@ -16,6 +17,8 @@ Multiple flags will create multiple node pools. For example: --nodes=1:10:CX41:NBG1:pool3 ``` +You can find a deployment sample under [examples/cluster-autoscaler-run-on-master.yaml](examples/cluster-autoscaler-run-on-master.yaml). Please be aware that you should change the values within this deployment to reflect your cluster. + # Development Make sure you're inside the root path of the [autoscaler diff --git a/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt b/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt new file mode 100644 index 000000000000..7192d5f43a95 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt @@ -0,0 +1,35 @@ +#!/bin/bash +export DEBIAN_FRONTEND=noninteractive +export HOME=/root/ +apt-get update && apt-get install -y apt-transport-https ca-certificates curl software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +cat </etc/apt/sources.list.d/kubernetes.list +deb https://apt.kubernetes.io/ kubernetes-xenial main +EOF +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get upgrade -y +apt-get install -y kubelet=1.20.1-00 kubeadm=1.20.1-00 kubectl=1.20.1-00 +apt-mark hold kubelet kubeadm kubectl +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" +apt update +apt install -y docker-ce +cat < /etc/docker/daemon.json < --token --discovery-token-ca-cert-hash sha256: diff --git a/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml new file mode 100644 index 000000000000..6b0aa5e0a242 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml @@ -0,0 +1,190 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes"] + verbs: ["watch", "list", "get"] + - apiGroups: ["batch", "extensions"] + resources: ["jobs"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"] + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + app: cluster-autoscaler +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8085' + spec: + serviceAccountName: cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + # Node affinity is used to force cluster-autoscaler to stick + # to the master node. This allows the cluster to reliably downscale + # to zero worker nodes when needed. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:latest # or your custom image + name: cluster-autoscaler + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi + command: + - ./cluster-autoscaler + - --cloud-provider=hetzner + - --stderrthreshold=info + - --nodes=1:10:CPX11:FSN1:pool1 + env: + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + - name: HCLOUD_CLOUD_INIT + value: +# - name: HCLOUD_SSH_KEY +# value: +# - name: HCLOUD_NETWORK +# value: + volumeMounts: + - name: ssl-certs + mountPath: /etc/ssl/certs/ca-certificates.crt + readOnly: true + imagePullPolicy: "Always" + imagePullSecrets: + - name: gitlab-registry + volumes: + - name: ssl-certs + hostPath: + path: "/etc/ssl/certs/ca-certificates.crt" diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go index cf6a9818221d..e76de37b6356 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go @@ -21,10 +21,11 @@ import ( "encoding/base64" "errors" "fmt" - apiv1 "k8s.io/api/core/v1" - "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" "os" "strings" + + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" ) var ( @@ -39,6 +40,8 @@ type hetznerManager struct { apiCallContext context.Context cloudInit string image string + sshKey *hcloud.SSHKey + network *hcloud.Network } func newManager() (*hetznerManager, error) { @@ -64,11 +67,33 @@ func newManager() (*hetznerManager, error) { return nil, fmt.Errorf("failed to parse cloud init error: %s", err) } + var network *hcloud.Network + networkName := os.Getenv("HCLOUD_NETWORK") + + var sshKey *hcloud.SSHKey + sshKeyName := os.Getenv("HCLOUD_SSH_KEY") + if sshKeyName != "" { + sshKey, _, err = client.SSHKey.Get(ctx, sshKeyName) + if err != nil { + return nil, fmt.Errorf("failed to get ssh key error: %s", err) + } + } + + if networkName != "" { + network, _, err = client.Network.Get(ctx, networkName) + if err != nil { + return nil, fmt.Errorf("failed to get network error: %s", err) + } + + } + m := &hetznerManager{ client: client, nodeGroups: make(map[string]*hetznerNodeGroup), cloudInit: string(cloudInit), image: image, + sshKey: sshKey, + network: network, apiCallContext: ctx, } diff --git a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go index addcf9e199e5..a620f068abd8 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go +++ b/cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go @@ -24,7 +24,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud" "k8s.io/klog/v2" - schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" "math/rand" "strconv" "sync" @@ -344,7 +344,7 @@ func serverTypeAvailable(manager *hetznerManager, instanceType string, region st func createServer(n *hetznerNodeGroup) error { StartAfterCreate := true - serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, hcloud.ServerCreateOpts{ + opts := hcloud.ServerCreateOpts{ Name: newNodeName(n), UserData: n.manager.cloudInit, Location: &hcloud.Location{Name: n.region}, @@ -354,10 +354,17 @@ func createServer(n *hetznerNodeGroup) error { Labels: map[string]string{ nodeGroupLabel: n.id, }, - }) + } + if n.manager.sshKey != nil { + opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey} + } + if n.manager.network != nil { + opts.Networks = []*hcloud.Network{n.manager.network} + } + serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, opts) if err != nil { - return fmt.Errorf("could not create server type %s in region %s", n.instanceType, n.region) + return fmt.Errorf("could not create server type %s in region %s: %v", n.instanceType, n.region, err) } server := serverCreateResult.Server diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py index 5dd5864a0474..b0800993085d 100755 --- a/hack/boilerplate/boilerplate.py +++ b/hack/boilerplate/boilerplate.py @@ -151,7 +151,8 @@ def file_extension(filename): "cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3", "cluster-autoscaler/cloudprovider/digitalocean/godo", "cluster-autoscaler/cloudprovider/magnum/gophercloud", - "cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go"] + "cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go", + "cluster-autoscaler/cloudprovider/hetzner/hcloud-go"] # list all the files contain 'DO NOT EDIT', but are not generated skipped_ungenerated_files = ['hack/build-ui.sh', 'hack/lib/swagger.sh', diff --git a/hack/verify-gofmt.sh b/hack/verify-gofmt.sh index 6b99e30f79f8..e620f78cd363 100755 --- a/hack/verify-gofmt.sh +++ b/hack/verify-gofmt.sh @@ -38,6 +38,7 @@ find_files() { -o -wholename './cluster-autoscaler/cloudprovider/digitalocean/godo/*' \ -o -wholename './cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3/*' \ -o -wholename './cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go/*' \ + -o -wholename './cluster-autoscaler/cloudprovider/hetzner/hcloud-go/*' \ \) -prune \ \) -name '*.go' } diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index 8b5d50304d17..db368e898179 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -30,6 +30,7 @@ excluded_packages=( 'cluster-autoscaler/cloudprovider/exoscale/internal' 'cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3' 'cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go' + 'cluster-autoscaler/cloudprovider/hetzner/hcloud-go' ) FIND_PACKAGES='go list ./... '