Support autoscaling

hack/stack.sh

@@ -74,25 +74,26 @@ newgrp docker <<EOF
 kind create cluster
 EOF
 
-# Install Go 1.19
-curl -Lo /tmp/go1.19.3.linux-amd64.tar.gz https://go.dev/dl/go1.19.3.linux-amd64.tar.gz
-rm -rf /usr/local/go && sudo tar -C /usr/local -xzf /tmp/go1.19.3.linux-amd64.tar.gz
-
-# Install drone/envsubst
-/usr/local/go/bin/go install github.com/drone/envsubst/v2/cmd/envsubst@latest
+# Label a control plane node
+kubectl label node kind-control-plane openstack-control-plane=enabled
 
 # Initialize the `clusterctl` CLI
 export EXP_CLUSTER_RESOURCE_SET=true
 export CLUSTER_TOPOLOGY=true
-curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0-beta.1/core-components.yaml | $HOME/go/bin/envsubst | kubectl apply -f-
-curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0-beta.1/control-plane-components.yaml | $HOME/go/bin/envsubst | kubectl apply -f-
-curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.3.0-beta.1/bootstrap-components.yaml | $HOME/go/bin/envsubst | kubectl apply -f-
-curl -L https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/components/nightly_main_20221109/infrastructure-components.yaml | $HOME/go/bin/envsubst | kubectl apply -f-
+clusterctl init \
+   --core cluster-api:v1.3.0-rc.0 \
+   --bootstrap kubeadm:v1.3.0-rc.0 \
+   --control-plane kubeadm:v1.3.0-rc.0
+curl -L https://storage.googleapis.com/artifacts.k8s-staging-capi-openstack.appspot.com/components/nightly_main_20221109/infrastructure-components.yaml | kubectl apply -f-
 
 # Install Skopeo
 sudo curl -Lo /usr/local/bin/skopeo https://github.com/lework/skopeo-binary/releases/download/v1.10.0/skopeo-linux-amd64
 sudo chmod +x /usr/local/bin/skopeo
 
+# Install Flux
+curl -s https://fluxcd.io/install.sh | sudo bash
+flux install
+
 # Install `magnum-cluster-api`
 pip install -U setuptools pip
 $HOME/.local/bin/pip3 install -e .

magnum_cluster_api/driver.py

@@ -186,6 +186,7 @@ def delete_cluster(self, context, cluster):
         resources.ClusterResourceSet(self.k8s_api, cluster).delete()
         resources.ClusterResourcesConfigMap(context, self.k8s_api, cluster).delete()
         resources.Cluster(context, self.k8s_api, cluster).delete()
+        resources.ClusterAutoscalerHelmRelease(self.k8s_api, cluster).delete()
 
     def create_nodegroup(self, context, cluster, nodegroup):
         resources.apply_cluster_from_magnum_cluster(context, self.k8s_api, cluster)

magnum_cluster_api/objects.py

@@ -65,3 +65,15 @@ class StorageClass(pykube.objects.APIObject):
     version = "storage.k8s.io/v1"
     endpoint = "storageclasses"
     kind = "StorageClass"
+
+
+class HelmRelease(pykube.objects.NamespacedAPIObject):
+    version = "helm.toolkit.fluxcd.io/v2beta1"
+    endpoint = "helmreleases"
+    kind = "HelmRelease"
+
+
+class HelmRepository(pykube.objects.NamespacedAPIObject):
+    version = "source.toolkit.fluxcd.io/v1beta2"
+    endpoint = "helmrepositories"
+    kind = "HelmRepository"

magnum_cluster_api/resources.py

@@ -20,6 +20,7 @@
 KUBE_TAG = "v1.25.3"
 CLOUD_PROVIDER_TAG = "v1.25.3"
 CALICO_TAG = "v3.24.2"
+AUTOSCALER_HELM_CHART_VERSION = "9.21.0"
 CSI_TAG = "v1.25.3"
 
 CLUSTER_CLASS_VERSION = pkg_resources.require("magnum_cluster_api")[0].version
@@ -85,6 +86,66 @@ def labels(self) -> dict:
         }
 
 
+class ClusterAutoscalerHelmRepository(Base):
+    def get_object(self) -> objects.HelmRepository:
+        return objects.HelmRepository(
+            self.api,
+            {
+                "apiVersion": objects.HelmRepository.version,
+                "kind": objects.HelmRepository.kind,
+                "metadata": {
+                    "name": "autoscaler",
+                    "namespace": "magnum-system",
+                },
+                "spec": {
+                    "interval": "1m",
+                    "url": "https://kubernetes.github.io/autoscaler",
+                },
+            },
+        )
+
+
+class ClusterAutoscalerHelmRelease(ClusterBase):
+    def get_object(self) -> objects.HelmRelease:
+        cluster_name = utils.get_or_generate_cluster_api_name(self.api, self.cluster)
+        return objects.HelmRelease(
+            self.api,
+            {
+                "apiVersion": objects.HelmRelease.version,
+                "kind": objects.HelmRelease.kind,
+                "metadata": {
+                    "name": cluster_name,
+                    "namespace": "magnum-system",
+                },
+                "spec": {
+                    "interval": "60s",
+                    "chart": {
+                        "spec": {
+                            "chart": "cluster-autoscaler",
+                            "version": AUTOSCALER_HELM_CHART_VERSION,
+                            "sourceRef": {
+                                "kind": objects.HelmRepository.kind,
+                                "name": "autoscaler",
+                            },
+                        },
+                    },
+                    "values": {
+                        "fullnameOverride": f"{cluster_name}-autoscaler",
+                        "cloudProvider": "clusterapi",
+                        "clusterAPIMode": "kubeconfig-incluster",
+                        "clusterAPIKubeconfigSecret": f"{cluster_name}-kubeconfig",
+                        "autoDiscovery": {
+                            "clusterName": cluster_name,
+                        },
+                        "nodeSelector": {
+                            "openstack-control-plane": "enabled",
+                        },
+                    },
+                },
+            },
+        )
+
+
 class ClusterResourcesConfigMap(ClusterBase):
     def __init__(
         self,
@@ -103,7 +164,6 @@ def get_object(self) -> pykube.ConfigMap:
         manifests_path = pkg_resources.resource_filename(
             "magnum_cluster_api", "manifests"
         )
-
         calico_version = utils.get_cluster_label(self.cluster, "calico_tag", CALICO_TAG)
         ccm_version = utils.get_cluster_label(
             self.cluster, "cloud_provider_tag", CLOUD_PROVIDER_TAG
@@ -201,6 +261,16 @@ def get_object(self) -> pykube.ConfigMap:
             },
         )
 
+    def get_or_none(self) -> objects.Cluster:
+        return pykube.ConfigMap.objects(
+            self.api, namespace="magnum-system"
+        ).get_or_none(name=self.cluster.uuid)
+
+    def delete(self):
+        cr_cm = self.get_or_none()
+        if cr_cm:
+            cr_cm.delete()
+
 
 class ClusterResourceSet(ClusterBase):
     def get_object(self) -> objects.ClusterResourceSet:
@@ -1073,6 +1143,9 @@ def get_or_none(self) -> objects.Cluster:
         )
 
     def get_object(self) -> objects.Cluster:
+        auto_scaling_enabled = utils.get_cluster_label_as_bool(
+            self.cluster, "auto_scaling_enabled", False
+        )
         return objects.Cluster(
             self.api,
             {
@@ -1120,7 +1193,25 @@ def get_object(self) -> objects.Cluster:
                                 {
                                     "class": "default-worker",
                                     "name": ng.name,
-                                    "replicas": ng.node_count,
+                                    "replicas": None
+                                    if auto_scaling_enabled
+                                    else ng.node_count,
+                                    "metadata": {
+                                        "annotations": {
+                                            "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size": utils.get_cluster_label(  # noqa: E501
+                                                self.cluster,
+                                                "min_node_count",
+                                                "1",
+                                            ),
+                                            "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size": utils.get_cluster_label(  # noqa: E501
+                                                self.cluster,
+                                                "max_node_count",
+                                                "1",
+                                            ),
+                                        }
+                                    }
+                                    if auto_scaling_enabled
+                                    else {},
                                     "failureDomain": utils.get_cluster_label(
                                         self.cluster, "availability_zone", ""
                                     ),
@@ -1320,6 +1411,9 @@ def apply_cluster_from_magnum_cluster(
     ClusterResourcesConfigMap(context, api, cluster).apply()
     ClusterResourceSet(api, cluster).apply()
     Cluster(context, api, cluster).apply()
+    ClusterAutoscalerHelmRepository(api).apply()
+    if utils.get_cluster_label_as_bool(cluster, "auto_scaling_enabled", False):
+        ClusterAutoscalerHelmRelease(api, cluster).apply()
 
 
 def get_kubeadm_control_plane(

magnum_cluster_api/utils.py

@@ -33,7 +33,7 @@ def generate_cluster_api_name(
     api: pykube.HTTPClient,
     cluster: magnum_objects.Cluster,
 ) -> str:
-    name = f"{cluster.name}-{shortuuid.uuid()[:10].lower()}"
+    name = f"{cluster.name}-{shortuuid.uuid()[:10].lower()}".replace(".", "-")
     if (
         objects.Cluster.objects(api)
         .filter(namespace="magnum-system")