Improve readme and add a sample

Signed-off-by: Lukas Kämmerling <[email protected]>

cluster-autoscaler/cloudprovider/hetzner/README.md

@@ -5,9 +5,10 @@ The cluster autoscaler for Hetzner Cloud scales worker nodes.
 # Configuration
 
 `HCLOUD_TOKEN` Required Hetzner Cloud token.
-`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster
+`HCLOUD_CLOUD_INIT` Base64 encoded Cloud Init yaml with commands to join the cluster, Sample [examples/cloud-init.txt for (Kubernetes 1.20.1)](examples/cloud-init.txt)
 `HCLOUD_IMAGE` Defaults to `ubuntu-20.04`, @see https://docs.hetzner.cloud/#images
-
+`HCLOUD_NETWORK` Default empty , The name of the network that is used in the cluster , @see https://docs.hetzner.cloud/#networks
+`HCLOUD_SSH_KEY` Default empty , This SSH Key will have access to the fresh created server, @see https://docs.hetzner.cloud/#ssh-keys
 Node groups must be defined with the `--nodes=<min-servers>:<max-servers>:<instance-type>:<region>:<name>` flag.
 Multiple flags will create multiple node pools. For example:
 ```
@@ -16,6 +17,8 @@ Multiple flags will create multiple node pools. For example:
 --nodes=1:10:CX41:NBG1:pool3
 ```
 
+You can find a deployment sample under [examples/cluster-autoscaler-run-on-master.yaml](examples/cluster-autoscaler-run-on-master.yaml). Please be aware that you should change the values within this deployment to reflect your cluster.
+
 # Development
 
 Make sure you're inside the root path of the [autoscaler

cluster-autoscaler/cloudprovider/hetzner/examples/cloud-init.txt

@@ -0,0 +1,35 @@
+#!/bin/bash
+export DEBIAN_FRONTEND=noninteractive
+export HOME=/root/
+apt-get update && apt-get install -y apt-transport-https ca-certificates curl software-properties-common
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
+deb https://apt.kubernetes.io/ kubernetes-xenial main
+EOF
+add-apt-repository   "deb [arch=amd64] https://download.docker.com/linux/ubuntu   $(lsb_release -cs)   stable"
+apt-get update
+apt-get upgrade -y
+apt-get install -y kubelet=1.20.1-00 kubeadm=1.20.1-00 kubectl=1.20.1-00
+apt-mark hold kubelet kubeadm kubectl
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable"
+apt update
+apt install -y docker-ce
+cat <<EOF | tee /etc/default/kubelet
+KUBELET_EXTRA_ARGS=--cloud-provider=external
+EOF
+cat > /etc/docker/daemon.json <<EOF
+{
+  "exec-opts": ["native.cgroupdriver=systemd"],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "100m"
+  },
+  "storage-driver": "overlay2"
+}
+EOF
+mkdir -p /etc/systemd/system/docker.service.d
+systemctl daemon-reload
+systemctl restart docker
+kubeadm join <master-ip> --token <token>  --discovery-token-ca-cert-hash sha256:<hash>

cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml

@@ -0,0 +1,190 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+  name: cluster-autoscaler
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["events", "endpoints"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/eviction"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    resourceNames: ["cluster-autoscaler"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update"]
+  - apiGroups: [""]
+    resources:
+      - "pods"
+      - "services"
+      - "replicationcontrollers"
+      - "persistentvolumeclaims"
+      - "persistentvolumes"
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["extensions"]
+    resources: ["replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["watch", "list"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses", "csinodes"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["batch", "extensions"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create"]
+  - apiGroups: ["coordination.k8s.io"]
+    resourceNames: ["cluster-autoscaler"]
+    resources: ["leases"]
+    verbs: ["get", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames: ["cluster-autoscaler-status", "cluster-autoscaler-priority-expander"]
+    verbs: ["delete", "get", "update", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    app: cluster-autoscaler
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+      annotations:
+        prometheus.io/scrape: 'true'
+        prometheus.io/port: '8085'
+    spec:
+      serviceAccountName: cluster-autoscaler
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/master
+      # Node affinity is used to force cluster-autoscaler to stick
+      # to the master node. This allows the cluster to reliably downscale
+      # to zero worker nodes when needed.
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/master
+                    operator: Exists
+      containers:
+        - image: k8s.gcr.io/autoscaling/cluster-autoscaler:latest  # or your custom image
+          name: cluster-autoscaler
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi
+          command:
+            - ./cluster-autoscaler
+            - --cloud-provider=hetzner
+            - --stderrthreshold=info
+            - --nodes=1:10:CPX11:FSN1:pool1
+          env:
+          - name: HCLOUD_TOKEN
+            valueFrom:
+                secretKeyRef:
+                  name: hcloud
+                  key: token
+          - name: HCLOUD_CLOUD_INIT
+            value: <your-cloud-init-data-base64-encoded>
+#         - name: HCLOUD_SSH_KEY
+#            value: <optional SSH Key Name or ID>
+#          - name: HCLOUD_NETWORK
+#            value: <optional Network Name or ID>
+          volumeMounts:
+            - name: ssl-certs
+              mountPath: /etc/ssl/certs/ca-certificates.crt
+              readOnly: true
+          imagePullPolicy: "Always"
+      imagePullSecrets:
+        - name: gitlab-registry
+      volumes:
+        - name: ssl-certs
+          hostPath:
+            path: "/etc/ssl/certs/ca-certificates.crt"

cluster-autoscaler/cloudprovider/hetzner/hetzner_manager.go

@@ -21,10 +21,11 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	apiv1 "k8s.io/api/core/v1"
-	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud"
 	"os"
 	"strings"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud"
 )
 
 var (
@@ -39,6 +40,8 @@ type hetznerManager struct {
 	apiCallContext context.Context
 	cloudInit      string
 	image          string
+	sshKey         *hcloud.SSHKey
+	network        *hcloud.Network
 }
 
 func newManager() (*hetznerManager, error) {
@@ -64,11 +67,33 @@ func newManager() (*hetznerManager, error) {
 		return nil, fmt.Errorf("failed to parse cloud init error: %s", err)
 	}
 
+	var network *hcloud.Network
+	networkName := os.Getenv("HCLOUD_NETWORK")
+
+	var sshKey *hcloud.SSHKey
+	sshKeyName := os.Getenv("HCLOUD_SSH_KEY")
+	if sshKeyName != "" {
+		sshKey, _, err = client.SSHKey.Get(ctx, sshKeyName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get ssh key error: %s", err)
+		}
+	}
+
+	if networkName != "" {
+		network, _, err = client.Network.Get(ctx, networkName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get network error: %s", err)
+		}
+
+	}
+
 	m := &hetznerManager{
 		client:         client,
 		nodeGroups:     make(map[string]*hetznerNodeGroup),
 		cloudInit:      string(cloudInit),
 		image:          image,
+		sshKey:         sshKey,
+		network:        network,
 		apiCallContext: ctx,
 	}
 

cluster-autoscaler/cloudprovider/hetzner/hetzner_node_group.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider/hetzner/hcloud-go/hcloud"
 	"k8s.io/klog/v2"
-	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1"
+	schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework"
 	"math/rand"
 	"strconv"
 	"sync"
@@ -344,7 +344,7 @@ func serverTypeAvailable(manager *hetznerManager, instanceType string, region st
 
 func createServer(n *hetznerNodeGroup) error {
 	StartAfterCreate := true
-	serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, hcloud.ServerCreateOpts{
+	opts := hcloud.ServerCreateOpts{
 		Name:             newNodeName(n),
 		UserData:         n.manager.cloudInit,
 		Location:         &hcloud.Location{Name: n.region},
@@ -354,10 +354,17 @@ func createServer(n *hetznerNodeGroup) error {
 		Labels: map[string]string{
 			nodeGroupLabel: n.id,
 		},
-	})
+	}
+	if n.manager.sshKey != nil {
+		opts.SSHKeys = []*hcloud.SSHKey{n.manager.sshKey}
+	}
+	if n.manager.network != nil {
+		opts.Networks = []*hcloud.Network{n.manager.network}
+	}
+	serverCreateResult, _, err := n.manager.client.Server.Create(n.manager.apiCallContext, opts)
 
 	if err != nil {
-		return fmt.Errorf("could not create server type %s in region %s", n.instanceType, n.region)
+		return fmt.Errorf("could not create server type %s in region %s: %v", n.instanceType, n.region, err)
 	}
 
 	server := serverCreateResult.Server