From 8a35d782536bffa41b8fdb03ce0e43a6cd6c1401 Mon Sep 17 00:00:00 2001 From: Johann Schley <65346790+oujonny@users.noreply.github.com> Date: Thu, 20 Jan 2022 21:31:09 +0100 Subject: [PATCH] add external hcloud cloud controller manager (#8440) --- inventory/sample/group_vars/all/all.yml | 2 +- inventory/sample/group_vars/all/hcloud.yml | 14 ++++ .../hcloud/defaults/main.yml | 14 ++++ .../hcloud/tasks/main.yml | 30 ++++++++ ...ontroller-manager-ds-with-networks.yaml.j2 | 69 +++++++++++++++++++ ...hcloud-cloud-controller-manager-ds.yaml.j2 | 61 ++++++++++++++++ ...external-hcloud-cloud-role-bindings.yml.j2 | 13 ++++ .../external-hcloud-cloud-secret.yml.j2 | 8 +++ ...ternal-hcloud-cloud-service-account.yml.j2 | 6 ++ .../external_cloud_controller/meta/main.yml | 10 +++ .../defaults/main/main.yaml | 15 ++++ 11 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 inventory/sample/group_vars/all/hcloud.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 create mode 100644 roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 diff --git a/inventory/sample/group_vars/all/all.yml b/inventory/sample/group_vars/all/all.yml index 97b73b51a00..eca3e6de9b0 100644 --- a/inventory/sample/group_vars/all/all.yml +++ b/inventory/sample/group_vars/all/all.yml @@ -54,7 +54,7 @@ loadbalancer_apiserver_healthcheck_port: 8081 # cloud_provider: ## When cloud_provider is set to 'external', you can set the cloud controller to deploy -## Supported cloud controllers are: 'openstack' and 'vsphere' +## Supported cloud controllers are: 'openstack', 'vsphere' and 'hcloud' ## When openstack or vsphere are used make sure to source in the required fields # external_cloud_provider: diff --git a/inventory/sample/group_vars/all/hcloud.yml b/inventory/sample/group_vars/all/hcloud.yml new file mode 100644 index 00000000000..ff90dcc8695 --- /dev/null +++ b/inventory/sample/group_vars/all/hcloud.yml @@ -0,0 +1,14 @@ +## Values for the external Hcloud Cloud Controller +# external_hcloud_cloud: +# hcloud_api_token: "" +# token_secret_name: hcloud +# +# service_account_name: cloud-controller-manager +# +# controller_image_tag: "latest" +# ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset +# ## Format: +# ## external_hcloud_cloud.controller_extra_args: +# ## arg1: "value1" +# ## arg2: "value2" +# controller_extra_args: {} diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml b/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml new file mode 100644 index 00000000000..5d9ba294bf4 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/defaults/main.yml @@ -0,0 +1,14 @@ +--- +external_hcloud_cloud: + hcloud_api_token: "" + token_secret_name: hcloud + + service_account_name: cloud-controller-manager + + controller_image_tag: "latest" + ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset + ## Format: + ## external_hcloud_cloud.controller_extra_args: + ## arg1: "value1" + ## arg2: "value2" + controller_extra_args: {} diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml b/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml new file mode 100644 index 00000000000..adaff22194a --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/tasks/main.yml @@ -0,0 +1,30 @@ +--- +- name: External Hcloud Cloud Controller | Generate Manifests + template: + src: "{{ item.file }}.j2" + dest: "{{ kube_config_dir }}/{{ item.file }}" + group: "{{ kube_cert_group }}" + mode: 0640 + with_items: + - {name: external-hcloud-cloud-secret, file: external-hcloud-cloud-secret.yml} + - {name: external-hcloud-cloud-service-account, file: external-hcloud-cloud-service-account.yml} + - {name: external-hcloud-cloud-role-bindings, file: external-hcloud-cloud-role-bindings.yml} + - {name: external-hcloud-cloud-controller-manager-ds, file: external-hcloud-cloud-controller-manager-ds.yml} + - {name: external-hcloud-cloud-controller-manager-ds-with-networks, file: external-hcloud-cloud-controller-manager-ds-with-networks.yml} + register: external_hcloud_manifests + when: inventory_hostname == groups['kube_control_plane'][0] + tags: external-hcloud + +- name: External Hcloud Cloud Controller | Apply Manifests + kube: + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/{{ item.item.file }}" + state: "latest" + with_items: + - "{{ external_hcloud_manifests.results }}" + when: + - inventory_hostname == groups['kube_control_plane'][0] + - not item is skipped + loop_control: + label: "{{ item.item.file }}" + tags: external-hcloud diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 new file mode 100644 index 00000000000..3bbe1075301 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds-with-networks.yaml.j2 @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: DeamonSet +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: hcloud-cloud-controller-manger +spec: + selector: + matchLabels: + app: hcloud-cloud-controller-manager + template: + metadata: + labels: + app: hcloud-cloud-controller-manager + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: {{ external_hcloud_cloud.service_account_name }} + dnsPolicy: Default + tolerations: + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + operator: Exists + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + hostNetwork: true + containers: + - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + - "--allocate-node-cidrs=true" + - "--cluster-cidr=10.244.0.0/16" + args: +{% for key, value in external_hcloud_cloud.controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + resources: + requests: + cpu: 100m + memory: 50Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: hcloud + key: token + - name: HCLOUD_NETWORK + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: {{ external_hcloud_cloud.token_secret_key }} diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 new file mode 100644 index 00000000000..fecee8d0afe --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-controller-manager-ds.yaml.j2 @@ -0,0 +1,61 @@ +--- +apiVersion: apps/v1 +kind: DeamonSet +metadata: + name: hcloud-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: hcloud-cloud-controller-manger +spec: + selector: + matchLabels: + app: hcloud-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + app: hcloud-cloud-controller-manager + annotations: + scheduler.alpha.kubernetes.io/critical-pod: '' + spec: + serviceAccountName: {{ external_hcloud_cloud.service_account_name }} + dnsPolicy: Default + tolerations: + - key: "node.cloudprovider.kubernetes.io/uninitialized" + value: "true" + effect: "NoSchedule" + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/master" + effect: NoSchedule + - key: "node-role.kubernetes.io/control-plane" + effect: NoSchedule + - key: "node.kubernetes.io/not-ready" + effect: "NoSchedule" + containers: + - image: {{ docker_image_repo }}/hetznercloud/hcloud-cloud-controller-manager:{{ external_hcloud_cloud.controller_image_tag }} + name: hcloud-cloud-controller-manager + command: + - "/bin/hcloud-cloud-controller-manager" + - "--cloud-provider=hcloud" + - "--leader-elect=false" + - "--allow-untagged-cloud" + args: +{% for key, value in external_hcloud_cloud.controller_extra_args.items() %} + - "{{ '--' + key + '=' + value }}" +{% endfor %} + resources: + requests: + cpu: 100m + memory: 50Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: HCLOUD_TOKEN + valueFrom: + secretKeyRef: + name: {{ external_hcloud_cloud.token_secret_name }} + key: {{ external_hcloud_cloud.token_secret_key }} diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 new file mode 100644 index 00000000000..270c947b585 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-role-bindings.yml.j2 @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: {{ external_hcloud_cloud.service_account_name }} + namespace: kube-system diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 new file mode 100644 index 00000000000..614d278978c --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-secret.yml.j2 @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: "{{ external_hcloud_cloud.token_secret_name }}" + namespace: kube-system +data: + token: "{{ external_hcloud_cloud.hcloud_api_token | base64 }}" diff --git a/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 new file mode 100644 index 00000000000..93277dddd30 --- /dev/null +++ b/roles/kubernetes-apps/external_cloud_controller/hcloud/templates/external-hcloud-cloud-service-account.yml.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ external_hcloud_cloud.service_account_name }} + namespace: kube-system diff --git a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml index a75a42995fe..6e8c235dd65 100644 --- a/roles/kubernetes-apps/external_cloud_controller/meta/main.yml +++ b/roles/kubernetes-apps/external_cloud_controller/meta/main.yml @@ -20,3 +20,13 @@ dependencies: tags: - external-cloud-controller - external-vsphere + - role: kubernetes-apps/external_cloud_controller/hcloud + when: + - cloud_provider is defined + - cloud_provider == "external" + - external_cloud_provider is defined + - external_cloud_provider == "hcloud" + - inventory_hostname == groups['kube_control_plane'][0] + tags: + - external-cloud-controller + - external-hcloud diff --git a/roles/kubespray-defaults/defaults/main/main.yaml b/roles/kubespray-defaults/defaults/main/main.yaml index 41e4a3f41b5..37253c4f02c 100644 --- a/roles/kubespray-defaults/defaults/main/main.yaml +++ b/roles/kubespray-defaults/defaults/main/main.yaml @@ -435,6 +435,21 @@ external_openstack_lbaas_use_octavia: false external_openstack_network_internal_networks: [] external_openstack_network_public_networks: [] +# Default values for the external Hcloud Cloud Controller +external_hcloud_cloud: + hcloud_api_token: "" + token_secret_name: hcloud + + service_account_name: cloud-controller-manager + + controller_image_tag: "latest" + ## A dictionary of extra arguments to add to the openstack cloud controller manager daemonset + ## Format: + ## external_hcloud_cloud.controller_extra_args: + ## arg1: "value1" + ## arg2: "value2" + controller_extra_args: {} + ## List of authorization modes that must be configured for ## the k8s cluster. Only 'AlwaysAllow', 'AlwaysDeny', 'Node' and ## 'RBAC' modes are tested. Order is important.