Merge pull request #692 from tserong/wip-longhorn

Add ability to deploy Longhorn on k3s

README.md

@@ -54,6 +54,7 @@ The Jenkins CI tests that `sesdev` can be used to deploy a single-node Ceph
       * [k3s (with or without Rook/Ceph/SES)](#k3s-with-or-without-rookcephses)
          * [k3s cluster](#k3s-cluster)
          * [k3s with Rook/Ceph/SES](#k3s-with-rookcephses)
+         * [k3s with Longhorn](#k3s-with-longhorn)
       * [On a remote libvirt server via SSH](#on-a-remote-libvirt-server-via-ssh)
       * [Using salt instead of DeepSea/ceph-salt CLI](#using-salt-instead-of-deepseaceph-salt-cli)
       * [With a FQDN environment](#with-a-fqdn-environment)
@@ -556,8 +557,8 @@ This uses `curl -sfL https://get.k3s.io | -` to install k3s,
 and `curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash` to install helm.
 
 By default it just creates and configures a k3s cluster, and workers don't
-have any disks unless the `--deploy-ses` (see below) or `--num-disks` options
-are given.
+have any disks unless the `--deploy-ses`, `--deploy-longhorn` (see below)
+or `--num-disks` options are given.
 
 ##### k3s with Rook/Ceph/SES
 
@@ -569,6 +570,28 @@ worker node 3:
 $ sesdev create k3s --deploy-ses
 ```
 
+##### k3s with Longhorn
+
+To have sesdev deploy [Longhorn](https://longhorn.io) instead of Ceph, give
+the `--deploy-longhorn` option.  By default this will deploy 4 worker nodes
+each with one additional 8G disk, mounted at /var/lib/longhorn, and will
+install the latest stable version of Longhorn:
+
+```
+$ sesdev create k3s --deploy-longhorn
+```
+
+To deploy a specific version of Longorn, use the `--longhorn-version` option:
+
+```
+$ sesdev create k3s --deploy-longhorn --longhorn-version=1.4.1
+```
+
+Currently Longhorn deployments will only use _one_ disk.  If more are
+specified using the `--num-disks` option, only the first disk will be
+mounted for use by Longhorn.  All other additional disks will remain
+untouched.
+
 #### On a remote libvirt server via SSH
 
 If you would like to start the cluster VMs on a remote server via libvirt/SSH,

sesdev/__init__.py

@@ -387,6 +387,7 @@ def _gen_settings_dict(
         deepsea_branch=None,
         deepsea_repo=None,
         deploy_ses=None,
+        deploy_longhorn=None,
         devel=None,
         disk_size=None,
         dry_run=None,
@@ -437,6 +438,7 @@ def _gen_settings_dict(
         msgr2_secure_mode=None,
         msgr2_prefer_secure=None,
         k3s_version=None,
+        longhorn_version=None,
 ):
 
     settings_dict = {}
@@ -665,6 +667,9 @@ def _gen_settings_dict(
     if deploy_ses and version == 'k3s':
         settings_dict['k3s_deploy_ses'] = True
 
+    if deploy_longhorn and version == 'k3s':
+        settings_dict['k3s_deploy_longhorn'] = True
+
     for folder in synced_folder:
         try:
             src, dst = folder.split(':')
@@ -705,6 +710,9 @@ def _gen_settings_dict(
     if k3s_version is not None:
         settings_dict['k3s_version'] = k3s_version
 
+    if longhorn_version is not None:
+        settings_dict['longhorn_version'] = longhorn_version
+
     return settings_dict
 
 
@@ -791,6 +799,17 @@ def _create_command(deployment_id, settings_dict):
                     click.echo()
                     click.echo("Inside the toolbox you can use the ceph CLI (`ceph status` etc.)")
                     click.echo()
+                if dep.settings.k3s_deploy_longhorn:
+                    click.echo("Longhorn will now be deploying, which may take some time.")
+                    click.echo("After logging into the cluster, try these:")
+                    click.echo()
+                    click.echo("  # kubectl get pods -n longhorn-system --watch")
+                    click.echo("  # kubectl get pods -n longhorn-system")
+                    click.echo()
+                    click.echo("The Longhorn UI will be accessible via any cluster IP address")
+                    click.echo("(see the `kubectl -n longhorn-system get ingress` output above).")
+                    click.echo("Note that no authentication is required.")
+                    click.echo()
             else:
                 click.echo("Or, access the Ceph Dashboard with:")
                 click.echo()
@@ -942,6 +961,10 @@ def caasp4(deployment_id, **kwargs):
 @libvirt_options
 @click.option("--deploy-ses", is_flag=True, default=False,
               help="Deploy SES using rook in k3s")
+@click.option("--deploy-longhorn", is_flag=True, default=False,
+              help="Deploy Longhorn on k3s")
+@click.option("--longhorn-version", default=None,
+              help='Longhorn version to install, e.g. "1.4.1" (defaults to latest stable)')
 @click.option("--k3s-version", default=None,
               help='k3s version to install (defaults to latest stable)')
 def k3s(deployment_id, **kwargs):
@@ -950,7 +973,12 @@ def k3s(deployment_id, **kwargs):
     """
     _prep_kwargs(kwargs)
     settings_dict = _gen_settings_dict('k3s', **kwargs)
-    deployment_id = _maybe_gen_dep_id('k3s', deployment_id, settings_dict)
+    default_dep_id = 'k3s'
+    if kwargs['deploy_ses']:
+        default_dep_id += '-ses'
+    if kwargs['deploy_longhorn']:
+        default_dep_id += '-longhorn'
+    deployment_id = _maybe_gen_dep_id(default_dep_id, deployment_id, settings_dict)
     _create_command(deployment_id, settings_dict)
 
 

seslib/deployment.py

@@ -265,16 +265,20 @@ def __maybe_tweak_roles(self):
     def __maybe_adjust_num_disks(self):
         single_node = self.settings.single_node or len(self.settings.roles) == 1
         storage_nodes = self.node_counts["storage"]
+        # pylint: disable=too-many-boolean-expressions
         if ((self.settings.version == 'caasp4' and self.settings.caasp_deploy_ses) or
-            (self.settings.version == 'k3s' and self.settings.k3s_deploy_ses)):
+            (self.settings.version == 'k3s' and self.settings.k3s_deploy_ses) or
+            (self.settings.version == 'k3s' and self.settings.k3s_deploy_longhorn)):
             if single_node:
                 storage_nodes = 1
             else:
                 storage_nodes = self.node_counts["worker"]
         Log.debug("__maybe_adjust_num_disks: storage_nodes == {}".format(storage_nodes))
         if not self.settings.explicit_num_disks and storage_nodes:
             new_num_disks = None
-            if storage_nodes == 1:
+            if self.settings.k3s_deploy_longhorn:
+                new_num_disks = 1
+            elif storage_nodes == 1:
                 new_num_disks = 4
             elif storage_nodes == 2:
                 new_num_disks = 3
@@ -423,6 +427,7 @@ def __generate_nodes(self):
                         node.ram = 2 * 2**10
                 if (self.settings.caasp_deploy_ses or
                    self.settings.k3s_deploy_ses or
+                   self.settings.k3s_deploy_longhorn or
                    self.settings.explicit_num_disks):
                     if 'worker' in node_roles or single_node:
                         for _ in range(self.settings.num_disks):
@@ -677,6 +682,7 @@ def _generate_vagrantfile(self):
             'node_manager': NodeManager(list(self.nodes.values())),
             'caasp_deploy_ses': self.settings.caasp_deploy_ses,
             'k3s_deploy_ses': self.settings.k3s_deploy_ses,
+            'k3s_deploy_longhorn': self.settings.k3s_deploy_longhorn,
             'synced_folders': self.settings.synced_folder,
             'makecheck_ceph_repo': self.settings.makecheck_ceph_repo,
             'makecheck_ceph_branch': self.settings.makecheck_ceph_branch,
@@ -698,7 +704,8 @@ def _generate_vagrantfile(self):
             'rgw_ssl': self.settings.rgw_ssl,
             'internal_media_repo': self.internal_media_repo,
             'developer_tools_repos': self.developer_tools_repos,
-            'k3s_version': self.settings.k3s_version
+            'k3s_version': self.settings.k3s_version,
+            'longhorn_version': self.settings.longhorn_version,
         }
 
         scripts = {}

seslib/settings.py

@@ -25,6 +25,11 @@
         'help': 'Deploy SES using rook in k3s',
         'default': False,
     },
+    'k3s_deploy_longhorn': {
+        'type': bool,
+        'help': 'Deploy Longhorn on k3s',
+        'default': False
+    },
     'ceph_salt_git_repo': {
         'type': str,
         'help': 'If set, it will install ceph-salt from this git repo',
@@ -135,6 +140,11 @@
         'help': 'k3s version to install (defaults to latest stable)',
         'default': '',
     },
+    'longhorn_version': {
+        'type': str,
+        'help': 'Longhorn version to install (defaults to latest stable)',
+        'default': '',
+    },
     'ceph_image_path': {
         'type': str,
         'help': 'Container image path for Ceph daemons',

seslib/templates/k3s/provision.sh.j2

@@ -7,6 +7,12 @@ zypper --non-interactive install curl openssl
 # need this to pick up kubectl, helm etc. which are installed to /usr/local/bin
 export PATH=$PATH:/usr/local/bin
 
+{% if k3s_deploy_longhorn %}
+zypper --non-interactive install nfs-client open-iscsi e2fsprogs
+systemctl enable iscsid
+systemctl start iscsid
+{% endif %} {# k3s_deploy_longhorn #}
+
 {% if k3s_version %}
 export INSTALL_K3S_VERSION={{ k3s_version }}
 {% endif %}
@@ -81,6 +87,10 @@ helm version
 # by default, which of course won't work.
 echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile.local
 
+# The earlier addition of KUBECONFIG to /etc/profile.local won't help
+# us in this current session, so also need to set it explicitly here.
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+
 {% if k3s_deploy_ses %}
 
 mkdir -p ~/cluster
@@ -93,9 +103,6 @@ tar -xzf rook-ceph-*.tgz
 
 kubectl create namespace rook-ceph
 
-# The earlier addition of KUBECONFIG to /etc/profile.local won't help
-# us in this current session, so need to set it explicitly here.
-export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
 helm install -n rook-ceph rook-ceph ./rook-ceph/
 
 echo "Waiting for the rook operator"
@@ -115,6 +122,42 @@ kubectl create -f rook-ceph/examples/toolbox.yaml
 
 {% endif %} {# k3s_deploy_ses #}
 
+{% if k3s_deploy_longhorn %}
+
+# Make longhorn create default disks on all nodes that aren't the master
+kubectl label node -l 'node-role.kubernetes.io/master!=true' node.longhorn.io/create-default-disk=true
+
+helm repo add longhorn https://charts.longhorn.io
+helm repo update
+helm install longhorn longhorn/longhorn \
+    --namespace longhorn-system \
+    --create-namespace {{ "--version {}".format(longhorn_version) if longhorn_version }} \
+    --set defaultSettings.createDefaultDiskLabeledNodes=true
+
+# On k3s this just gives a traefik ingress for the UI with no authentication
+echo "
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: longhorn-ingress
+  namespace: longhorn-system
+spec:
+  rules:
+  - http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: longhorn-frontend
+            port:
+              number: 80
+" | kubectl -n longhorn-system create -f -
+
+kubectl -n longhorn-system get ingress
+
+{% endif %} {# k3s_deploy_longhorn #}
+
 {% else %} {# node == master #}
 
 function get_k3s_token {
@@ -148,4 +191,15 @@ rm /tmp/k3s_token
 
 curl -sfL https://get.k3s.io | K3S_URL=https://{{ master.fqdn }}:6443 sh -
 
+{% if k3s_deploy_longhorn %}
+if [ ! -b /dev/vdb ]; then
+    echo "ERROR: Longhorn deployments require one additional disk"
+    false
+fi
+mkfs.ext4 /dev/vdb
+mkdir /var/lib/longhorn
+echo "/dev/vdb /var/lib/longhorn ext4 defaults 0 2" >> /etc/fstab
+mount /var/lib/longhorn
+{% endif %} {# k3s_deploy_longhorn #}
+
 {% endif %}