From b24331b687e4050f1dadc78cefb6f5c6e66d03ad Mon Sep 17 00:00:00 2001 From: Tim Serong Date: Wed, 19 Apr 2023 15:12:02 +1000 Subject: [PATCH 1/3] k3s: append "-ses" to deployment ID if --deploy-ses is specified Signed-off-by: Tim Serong --- sesdev/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sesdev/__init__.py b/sesdev/__init__.py index a095e3f8..68365cb0 100644 --- a/sesdev/__init__.py +++ b/sesdev/__init__.py @@ -950,7 +950,10 @@ def k3s(deployment_id, **kwargs): """ _prep_kwargs(kwargs) settings_dict = _gen_settings_dict('k3s', **kwargs) - deployment_id = _maybe_gen_dep_id('k3s', deployment_id, settings_dict) + default_dep_id = 'k3s' + if kwargs['deploy_ses']: + default_dep_id += '-ses' + deployment_id = _maybe_gen_dep_id(default_dep_id, deployment_id, settings_dict) _create_command(deployment_id, settings_dict) From 077ec60d8d0ff9dd5ec80fcdce8e338a6ce41658 Mon Sep 17 00:00:00 2001 From: Tim Serong Date: Wed, 19 Apr 2023 15:14:07 +1000 Subject: [PATCH 2/3] Add ability to deploy Longhorn on k3s This adds the `--deploy-longhorn` option to `sesdev create k3s`, which will make sesdev deploy Longhorn instead of Ceph. By default it will deploy the latest released version of Longhorn, but this can be overridden by using the --longhorn-version option to specify a tag from https://github.com/longhorn/longhorn/tags. When deploying Longhorn, one extra 8GB disk will be added to each worker node mounted as ext4 under /var/lib/longhorn. One area for future work is to make it use multiple disks per node. The Longhorn UI will be made available via HTTP with no authentication. Signed-off-by: Tim Serong --- README.md | 28 +++++++++++- sesdev/__init__.py | 25 ++++++++++ seslib/deployment.py | 13 ++++-- seslib/settings.py | 10 ++++ seslib/templates/k3s/provision.sh.j2 | 68 ++++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9adf715f..8cbbc7ae 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ The Jenkins CI tests that `sesdev` can be used to deploy a single-node Ceph * [k3s (with or without Rook/Ceph/SES)](#k3s-with-or-without-rookcephses) * [k3s cluster](#k3s-cluster) * [k3s with Rook/Ceph/SES](#k3s-with-rookcephses) + * [k3s with Longhorn](#k3s-with-longhorn) * [On a remote libvirt server via SSH](#on-a-remote-libvirt-server-via-ssh) * [Using salt instead of DeepSea/ceph-salt CLI](#using-salt-instead-of-deepseaceph-salt-cli) * [With a FQDN environment](#with-a-fqdn-environment) @@ -556,8 +557,8 @@ This uses `curl -sfL https://get.k3s.io | -` to install k3s, and `curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash` to install helm. By default it just creates and configures a k3s cluster, and workers don't -have any disks unless the `--deploy-ses` (see below) or `--num-disks` options -are given. +have any disks unless the `--deploy-ses`, `--deploy-longhorn` (see below) +or `--num-disks` options are given. ##### k3s with Rook/Ceph/SES @@ -569,6 +570,29 @@ worker node 3: $ sesdev create k3s --deploy-ses ``` +##### k3s with Longhorn + +To have sesdev deploy [Longhorn](https://longhorn.io) instead of Ceph, give +the `--deploy-longhorn` option. By default this will deploy 4 worker nodes +each with one additional 8G disk, mounted at /var/lib/longhorn, and will +install the latest stable version of Longhorn: + +``` +$ sesdev create k3s --deploy-longhorn +``` + +To deploy a specific version of Longorn, use the `--longhorn-version` option +to specify a tag name from https://github.com/longhorn/longhorn/tags, e.g.: + +``` +$ sesdev create k3s --deploy-longhorn --longhorn-version=v1.4.1 +``` + +Currently Longhorn deployments will only use _one_ disk. If more are +specified using the `--num-disks` option, only the first disk will be +mounted for use by Longhorn. All other additional disks will remain +untouched. + #### On a remote libvirt server via SSH If you would like to start the cluster VMs on a remote server via libvirt/SSH, diff --git a/sesdev/__init__.py b/sesdev/__init__.py index 68365cb0..501859e4 100644 --- a/sesdev/__init__.py +++ b/sesdev/__init__.py @@ -387,6 +387,7 @@ def _gen_settings_dict( deepsea_branch=None, deepsea_repo=None, deploy_ses=None, + deploy_longhorn=None, devel=None, disk_size=None, dry_run=None, @@ -437,6 +438,7 @@ def _gen_settings_dict( msgr2_secure_mode=None, msgr2_prefer_secure=None, k3s_version=None, + longhorn_version=None, ): settings_dict = {} @@ -665,6 +667,9 @@ def _gen_settings_dict( if deploy_ses and version == 'k3s': settings_dict['k3s_deploy_ses'] = True + if deploy_longhorn and version == 'k3s': + settings_dict['k3s_deploy_longhorn'] = True + for folder in synced_folder: try: src, dst = folder.split(':') @@ -705,6 +710,9 @@ def _gen_settings_dict( if k3s_version is not None: settings_dict['k3s_version'] = k3s_version + if longhorn_version is not None: + settings_dict['longhorn_version'] = longhorn_version + return settings_dict @@ -791,6 +799,17 @@ def _create_command(deployment_id, settings_dict): click.echo() click.echo("Inside the toolbox you can use the ceph CLI (`ceph status` etc.)") click.echo() + if dep.settings.k3s_deploy_longhorn: + click.echo("Longhorn will now be deploying, which may take some time.") + click.echo("After logging into the cluster, try these:") + click.echo() + click.echo(" # kubectl get pods -n longhorn-system --watch") + click.echo(" # kubectl get pods -n longhorn-system") + click.echo() + click.echo("The Longhorn UI will be accessible via any cluster IP address") + click.echo("(see the `kubectl -n longhorn-system get ingress` output above).") + click.echo("Note that no authentication is required.") + click.echo() else: click.echo("Or, access the Ceph Dashboard with:") click.echo() @@ -942,6 +961,10 @@ def caasp4(deployment_id, **kwargs): @libvirt_options @click.option("--deploy-ses", is_flag=True, default=False, help="Deploy SES using rook in k3s") +@click.option("--deploy-longhorn", is_flag=True, default=False, + help="Deploy Longhorn on k3s") +@click.option("--longhorn-version", default=None, + help='Longhorn version to install, e.g. "v1.4.1" (defaults to latest stable)') @click.option("--k3s-version", default=None, help='k3s version to install (defaults to latest stable)') def k3s(deployment_id, **kwargs): @@ -953,6 +976,8 @@ def k3s(deployment_id, **kwargs): default_dep_id = 'k3s' if kwargs['deploy_ses']: default_dep_id += '-ses' + if kwargs['deploy_longhorn']: + default_dep_id += '-longhorn' deployment_id = _maybe_gen_dep_id(default_dep_id, deployment_id, settings_dict) _create_command(deployment_id, settings_dict) diff --git a/seslib/deployment.py b/seslib/deployment.py index 289f0208..fba687d0 100644 --- a/seslib/deployment.py +++ b/seslib/deployment.py @@ -265,8 +265,10 @@ def __maybe_tweak_roles(self): def __maybe_adjust_num_disks(self): single_node = self.settings.single_node or len(self.settings.roles) == 1 storage_nodes = self.node_counts["storage"] + # pylint: disable=too-many-boolean-expressions if ((self.settings.version == 'caasp4' and self.settings.caasp_deploy_ses) or - (self.settings.version == 'k3s' and self.settings.k3s_deploy_ses)): + (self.settings.version == 'k3s' and self.settings.k3s_deploy_ses) or + (self.settings.version == 'k3s' and self.settings.k3s_deploy_longhorn)): if single_node: storage_nodes = 1 else: @@ -274,7 +276,9 @@ def __maybe_adjust_num_disks(self): Log.debug("__maybe_adjust_num_disks: storage_nodes == {}".format(storage_nodes)) if not self.settings.explicit_num_disks and storage_nodes: new_num_disks = None - if storage_nodes == 1: + if self.settings.k3s_deploy_longhorn: + new_num_disks = 1 + elif storage_nodes == 1: new_num_disks = 4 elif storage_nodes == 2: new_num_disks = 3 @@ -423,6 +427,7 @@ def __generate_nodes(self): node.ram = 2 * 2**10 if (self.settings.caasp_deploy_ses or self.settings.k3s_deploy_ses or + self.settings.k3s_deploy_longhorn or self.settings.explicit_num_disks): if 'worker' in node_roles or single_node: for _ in range(self.settings.num_disks): @@ -677,6 +682,7 @@ def _generate_vagrantfile(self): 'node_manager': NodeManager(list(self.nodes.values())), 'caasp_deploy_ses': self.settings.caasp_deploy_ses, 'k3s_deploy_ses': self.settings.k3s_deploy_ses, + 'k3s_deploy_longhorn': self.settings.k3s_deploy_longhorn, 'synced_folders': self.settings.synced_folder, 'makecheck_ceph_repo': self.settings.makecheck_ceph_repo, 'makecheck_ceph_branch': self.settings.makecheck_ceph_branch, @@ -698,7 +704,8 @@ def _generate_vagrantfile(self): 'rgw_ssl': self.settings.rgw_ssl, 'internal_media_repo': self.internal_media_repo, 'developer_tools_repos': self.developer_tools_repos, - 'k3s_version': self.settings.k3s_version + 'k3s_version': self.settings.k3s_version, + 'longhorn_version': self.settings.longhorn_version, } scripts = {} diff --git a/seslib/settings.py b/seslib/settings.py index f97bdee5..eaf8b865 100644 --- a/seslib/settings.py +++ b/seslib/settings.py @@ -25,6 +25,11 @@ 'help': 'Deploy SES using rook in k3s', 'default': False, }, + 'k3s_deploy_longhorn': { + 'type': bool, + 'help': 'Deploy Longhorn on k3s', + 'default': False + }, 'ceph_salt_git_repo': { 'type': str, 'help': 'If set, it will install ceph-salt from this git repo', @@ -135,6 +140,11 @@ 'help': 'k3s version to install (defaults to latest stable)', 'default': '', }, + 'longhorn_version': { + 'type': str, + 'help': 'Longhorn version to install (defaults to latest stable)', + 'default': '', + }, 'ceph_image_path': { 'type': str, 'help': 'Container image path for Ceph daemons', diff --git a/seslib/templates/k3s/provision.sh.j2 b/seslib/templates/k3s/provision.sh.j2 index 2f241867..c1180566 100644 --- a/seslib/templates/k3s/provision.sh.j2 +++ b/seslib/templates/k3s/provision.sh.j2 @@ -7,6 +7,12 @@ zypper --non-interactive install curl openssl # need this to pick up kubectl, helm etc. which are installed to /usr/local/bin export PATH=$PATH:/usr/local/bin +{% if k3s_deploy_longhorn %} +zypper --non-interactive install nfs-client open-iscsi e2fsprogs +systemctl enable iscsid +systemctl start iscsid +{% endif %} {# k3s_deploy_longhorn #} + {% if k3s_version %} export INSTALL_K3S_VERSION={{ k3s_version }} {% endif %} @@ -115,6 +121,57 @@ kubectl create -f rook-ceph/examples/toolbox.yaml {% endif %} {# k3s_deploy_ses #} +{% if k3s_deploy_longhorn %} + +{% if longhorn_version %} +longhorn_version={{ longhorn_version }} +{% else %} +longhorn_version=$( + curl -sL \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version:2022-11-28" \ + https://api.github.com/repos/longhorn/longhorn/releases | \ + jq -r '.[].tag_name' | \ + grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | \ + sort --version-sort --reverse | \ + head -n 1 +) +{% endif %} + +curl -sSfL https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_version}/scripts/environment_check.sh | bash + +curl -sSLO https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_version}/deploy/longhorn.yaml + +# Make longhorn create default disks on all nodes that aren't the master +sed -i '/default-setting\.yaml: |-/a \ \ \ \ create-default-disk-labeled-nodes: true' longhorn.yaml +kubectl label node -l 'node-role.kubernetes.io/master!=true' node.longhorn.io/create-default-disk=true + +kubectl apply -f longhorn.yaml + +# On k3s this just gives a traefik ingress for the UI with no authentication +echo " +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: longhorn-ingress + namespace: longhorn-system +spec: + rules: + - http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: longhorn-frontend + port: + number: 80 +" | kubectl -n longhorn-system create -f - + +kubectl -n longhorn-system get ingress + +{% endif %} {# k3s_deploy_longhorn #} + {% else %} {# node == master #} function get_k3s_token { @@ -148,4 +205,15 @@ rm /tmp/k3s_token curl -sfL https://get.k3s.io | K3S_URL=https://{{ master.fqdn }}:6443 sh - +{% if k3s_deploy_longhorn %} +if [ ! -b /dev/vdb ]; then + echo "ERROR: Longhorn deployments require one additional disk" + false +fi +mkfs.ext4 /dev/vdb +mkdir /var/lib/longhorn +echo "/dev/vdb /var/lib/longhorn ext4 defaults 0 2" >> /etc/fstab +mount /var/lib/longhorn +{% endif %} {# k3s_deploy_longhorn #} + {% endif %} From a3f5bf1ea70fbf2fa9bf0091eb8d63f70d5c4469 Mon Sep 17 00:00:00 2001 From: Tim Serong Date: Thu, 20 Apr 2023 16:09:57 +1000 Subject: [PATCH 3/3] k3s/longhorn: deploy using helm instead of kubectl If we use helm to deploy Longhorn, we don't need to manually parse tags from the github API to figure out what the latest released version is - it will simply install the latest version by default. Signed-off-by: Tim Serong --- README.md | 5 ++-- sesdev/__init__.py | 2 +- seslib/templates/k3s/provision.sh.j2 | 34 ++++++++-------------------- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 8cbbc7ae..8cc7d02a 100644 --- a/README.md +++ b/README.md @@ -581,11 +581,10 @@ install the latest stable version of Longhorn: $ sesdev create k3s --deploy-longhorn ``` -To deploy a specific version of Longorn, use the `--longhorn-version` option -to specify a tag name from https://github.com/longhorn/longhorn/tags, e.g.: +To deploy a specific version of Longorn, use the `--longhorn-version` option: ``` -$ sesdev create k3s --deploy-longhorn --longhorn-version=v1.4.1 +$ sesdev create k3s --deploy-longhorn --longhorn-version=1.4.1 ``` Currently Longhorn deployments will only use _one_ disk. If more are diff --git a/sesdev/__init__.py b/sesdev/__init__.py index 501859e4..5b365385 100644 --- a/sesdev/__init__.py +++ b/sesdev/__init__.py @@ -964,7 +964,7 @@ def caasp4(deployment_id, **kwargs): @click.option("--deploy-longhorn", is_flag=True, default=False, help="Deploy Longhorn on k3s") @click.option("--longhorn-version", default=None, - help='Longhorn version to install, e.g. "v1.4.1" (defaults to latest stable)') + help='Longhorn version to install, e.g. "1.4.1" (defaults to latest stable)') @click.option("--k3s-version", default=None, help='k3s version to install (defaults to latest stable)') def k3s(deployment_id, **kwargs): diff --git a/seslib/templates/k3s/provision.sh.j2 b/seslib/templates/k3s/provision.sh.j2 index c1180566..f45a5d2e 100644 --- a/seslib/templates/k3s/provision.sh.j2 +++ b/seslib/templates/k3s/provision.sh.j2 @@ -87,6 +87,10 @@ helm version # by default, which of course won't work. echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile.local +# The earlier addition of KUBECONFIG to /etc/profile.local won't help +# us in this current session, so also need to set it explicitly here. +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + {% if k3s_deploy_ses %} mkdir -p ~/cluster @@ -99,9 +103,6 @@ tar -xzf rook-ceph-*.tgz kubectl create namespace rook-ceph -# The earlier addition of KUBECONFIG to /etc/profile.local won't help -# us in this current session, so need to set it explicitly here. -export KUBECONFIG=/etc/rancher/k3s/k3s.yaml helm install -n rook-ceph rook-ceph ./rook-ceph/ echo "Waiting for the rook operator" @@ -123,30 +124,15 @@ kubectl create -f rook-ceph/examples/toolbox.yaml {% if k3s_deploy_longhorn %} -{% if longhorn_version %} -longhorn_version={{ longhorn_version }} -{% else %} -longhorn_version=$( - curl -sL \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version:2022-11-28" \ - https://api.github.com/repos/longhorn/longhorn/releases | \ - jq -r '.[].tag_name' | \ - grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | \ - sort --version-sort --reverse | \ - head -n 1 -) -{% endif %} - -curl -sSfL https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_version}/scripts/environment_check.sh | bash - -curl -sSLO https://raw.githubusercontent.com/longhorn/longhorn/${longhorn_version}/deploy/longhorn.yaml - # Make longhorn create default disks on all nodes that aren't the master -sed -i '/default-setting\.yaml: |-/a \ \ \ \ create-default-disk-labeled-nodes: true' longhorn.yaml kubectl label node -l 'node-role.kubernetes.io/master!=true' node.longhorn.io/create-default-disk=true -kubectl apply -f longhorn.yaml +helm repo add longhorn https://charts.longhorn.io +helm repo update +helm install longhorn longhorn/longhorn \ + --namespace longhorn-system \ + --create-namespace {{ "--version {}".format(longhorn_version) if longhorn_version }} \ + --set defaultSettings.createDefaultDiskLabeledNodes=true # On k3s this just gives a traefik ingress for the UI with no authentication echo "