Support manila fileshares (cephfs)

ansible/fatimage.yml

@@ -41,10 +41,16 @@
         tasks_from: client-install.yml
       when: "'freeipa_client' in group_names"
 
-    # - import_playbook: filesystems.yml
-    - name: nfs
+    # - import_playbook: filesystems.yml:
+    - name: Install nfs packages
       dnf:
         name: nfs-utils
+      when: "'nfs' in group_names"
+    - name: Install Manila client packages
+      include_role:
+        name: stackhpc.os-manila-mount
+        tasks_from: install.yml
+      when: "'manila' in group_names"
 
 - import_playbook: extras.yml
 

ansible/filesystems.yml

@@ -16,3 +16,11 @@
   tasks:
     - include_role:
         name: stackhpc.nfs
+
+- name: Setup Manila share mounts
+  hosts: manila
+  become: true
+  tags: manila
+  tasks:
+    - include_role:
+        name: stackhpc.os-manila-mount

ansible/roles/cluster_infra/templates/resources.tf.j2

@@ -79,6 +79,22 @@ resource "openstack_blockstorage_volume_v3" "state" {
     size = "{{ state_volume_size }}"
 }
 
+{% if cluster_home_manila_share | bool %}
+resource "openstack_sharedfilesystem_share_v2" "home" {
+  name = "{{ cluster_name }}-home"
+  description = "Home for cluster"
+  share_proto = "CEPHFS"
+  share_type = {{ '"' + cluster_home_manila_share_type + '"' | default('null') }}
+  size = "{{ home_volume_size }}"
+}
+
+resource "openstack_sharedfilesystem_share_access_v2" "home" {
+  share_id     = openstack_sharedfilesystem_share_v2.home.id
+  access_type  = "cephx"
+  access_to    = "cluster_{{ cluster_id }}"
+  access_level = "rw"
+}
+{% else %}
 resource "openstack_blockstorage_volume_v3" "home" {
     name = "{{ cluster_name }}-home"
     description = "Home for control node"
@@ -89,6 +105,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
     {% endif %}
     {% endif %}
 }
+{% endif %}
 
 ######
 ###### Cluster network
@@ -334,13 +351,15 @@ resource "openstack_compute_instance_v2" "control" {
       uuid = openstack_blockstorage_volume_v3.state.id
   }
 
+  {% if not cluster_home_manila_share | bool %}
   # home volume:
   block_device {
       destination_type = "volume"
       source_type  = "volume"
       boot_index = -1
       uuid = openstack_blockstorage_volume_v3.home.id
   }
+  {% endif %}
 
   # Use cloud-init to a) inject SSH keys b) configure volumes
   user_data = <<-EOF
@@ -359,12 +378,14 @@ resource "openstack_compute_instance_v2" "control" {
       - {{ ssh_key }}
     {%- endfor %}
     bootcmd:
-    %{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
+    %{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
     - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
     %{endfor}
     mounts:
         - [LABEL=state, {{ appliances_state_dir }}, auto]
+        {% if not cluster_home_manila_share | bool %}
         - [LABEL=home, /exports/home, auto]
+        {% endif %}
   EOF
 }
 

environments/.caas/inventory/extra_groups

@@ -7,3 +7,7 @@ cluster
 [zenith:children]
 grafana
 openondemand
+
+[manila:children]
+login
+compute

environments/.caas/inventory/group_vars/all/cluster.yml

@@ -20,3 +20,7 @@ openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_ga
 openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"
 
 appliances_state_dir: /var/lib/state
+
+# Defaults for caas-provided extravars:
+cluster_project_manila_share: false
+cluster_home_manila_share: "{{ cluster_project_manila_share }}"

environments/.caas/inventory/group_vars/all/manila.yml

@@ -0,0 +1,16 @@
+caas_manila_home:
+  share_name: "{{ cluster_name }}-home"
+  mount_path: /home
+  mount_user: root
+  mount_group: root
+  mount_mode: u=rwX,go=rX
+
+cluster_project_manila_share_name: azimuth-project-share
+caas_manila_project:
+  share_name: "{{ cluster_project_manila_share_name }}"
+  mount_path: /project
+  mount_user: root
+  mount_group: root
+  mount_mode: ugo=rwX
+
+os_manila_mount_shares: "{{ ([caas_manila_home] if cluster_home_manila_share | bool else []) + ([caas_manila_project] if cluster_project_manila_share | bool else []) }}"

environments/.caas/inventory/group_vars/all/nfs.yml

@@ -1,16 +1,20 @@
 nfs_server: "{{ nfs_server_default }}"
 
-nfs_configurations:
-  - comment: Export /exports/home from Slurm control node as /home
-    nfs_enable:
-        server:  "{{ inventory_hostname in groups['control'] }}"
-        clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
-    nfs_export: "/exports/home" # assumes skeleton TF is being used
-    nfs_client_mnt_point: "/home"
+caas_nfs_ood_state:
   - comment: Export /var/lib/state from Slurm control node to OOD
     nfs_enable:
         server:  "{{ inventory_hostname in groups['control'] }}"
         clients: "{{ inventory_hostname in groups['openondemand'] }}"
     nfs_export: "{{ appliances_state_dir }}"
     nfs_client_mnt_point: "{{ appliances_state_dir }}"
     nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
+
+caas_nfs_home:
+  - comment: Export /exports/home from Slurm control node as /home
+    nfs_enable:
+        server:  "{{ inventory_hostname in groups['control'] }}"
+        clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
+    nfs_export: "/exports/home" # assumes skeleton TF is being used
+    nfs_client_mnt_point: "/home"
+
+nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"

environments/.caas/ui-meta/slurm-infra-manila.yml

@@ -0,0 +1,105 @@
+name: "slurm-manila-preview"
+label: "Slurm: Manila Integration Preview"
+description: >- 
+  Batch cluster running the Slurm workload manager, the Open 
+  OnDemand web interface, and custom monitoring.
+
+  Has /project and /home from CephFS/Manila.
+logo: https://upload.wikimedia.org/wikipedia/commons/thumb/3/3a/Slurm_logo.svg/158px-Slurm_logo.svg.png
+
+parameters:
+  - name: cluster_floating_ip
+    label: External IP
+    description: The external IP to use for the login node.
+    kind: cloud.ip
+    immutable: true
+
+  - name: compute_count
+    label: Compute node count
+    description: The number of compute nodes in the cluster.
+    kind: integer
+    options:
+      min: 1
+    default: 3
+
+  - name: compute_flavor
+    label: Compute node size
+    description: The size to use for the compute node.
+    kind: "cloud.size"
+    immutable: true
+    options:
+      min_ram: 2048
+      min_disk: 20
+
+  - name: home_volume_size
+    label: Home filesystem size (GB)
+    description: The size of the cloud volume or share to use for home directories
+    kind: integer
+    immutable: true
+    options:
+      min: 10
+    default: 100
+
+  - name: metrics_db_maximum_size
+    label: Metrics database size (GB)
+    description: |
+      The oldest metrics records in the [Prometheus](https://prometheus.io/) database will be 
+      discarded to ensure that the database does not grow larger than this size.
+
+      **A cloud volume of this size +10GB will be created to hold and persist the metrics 
+      database and important Slurm files.**
+    kind: integer
+    immutable: true
+    options:
+      min: 10
+    default: 10
+
+  - name: cluster_run_validation
+    label: Post-configuration validation
+    description: >-
+      If selected, post-configuration jobs will be executed to validate the core functionality
+      of the cluster when it is re-configured.
+    kind: boolean
+    required: false
+    default: true
+    options:
+      checkboxLabel: Run post-configuration validation?
+
+usage_template: |-
+  # Accessing the cluster using Open OnDemand
+
+  [Open OnDemand](https://openondemand.org/) is a web portal for managing HPC jobs, including graphical
+  environments such as [Jupyter Notebooks](https://jupyter.org/).
+
+  {% if cluster.outputs.openondemand_url %}
+  The Open OnDemand portal for this cluster is available at
+  [{{ cluster.outputs.openondemand_url.slice(8) }}]({{ cluster.outputs.openondemand_url }}).
+
+  Enter the username `azimuth` and password `{{ cluster.outputs.azimuth_user_password }}` when prompted.
+  {% else %}
+  The Open OnDemand portal for this cluster can be accessed from the services list.
+  {% endif %}
+
+  # Accessing the cluster using SSH
+
+  The cluster can be accessed over SSH via the external IP. The SSH public key of the user that
+  deployed the cluster is injected into the `azimuth` user:
+
+  ```
+  $ ssh azimuth@{{ cluster.outputs.cluster_access_ip | default('[cluster ip]') }}
+  [azimuth@{{ cluster.name }}-login-0 ~]$ sinfo
+  PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
+  compute*     up 60-00:00:0    {{ "%3s" | format(cluster.parameter_values.compute_count) }}   idle {{ cluster.name }}-compute-[0-{{ cluster.parameter_values.compute_count - 1 }}]
+  ```
+
+  The `rocky` user can be accessed the same way and has passwordless `sudo` enabled.
+
+  SSH access can be granted to additional users by placing their SSH public key in `~azimuth/.ssh/authorized_keys`.
+
+services:
+  - name: ood
+    label: Open OnDemand
+    icon_url: https://github.com/stackhpc/ansible-slurm-appliance/raw/main/environments/.caas/assets/ood-icon.png
+  - name: monitoring
+    label: Monitoring
+    icon_url: https://raw.githubusercontent.com/cncf/artwork/master/projects/prometheus/icon/color/prometheus-icon-color.png

environments/.stackhpc/inventory/extra_groups

@@ -22,3 +22,8 @@ cluster
 # [resolv_conf:children]
 # freeipa_client
 # --- end of FreeIPA example ---
+
+[manila:children]
+# Allows demo; also installs manila client in fat image
+login
+compute

environments/.stackhpc/terraform/main.tf

@@ -1,5 +1,14 @@
 # This terraform configuration uses the "skeleton" terraform, so that is checked by CI.
 
+terraform {
+  required_version = ">= 0.14"
+  required_providers {
+    openstack = {
+      source = "terraform-provider-openstack/openstack"
+    }
+  }
+}
+
 variable "environment_root" {
     type = string
     description = "Path to environment root, automatically set by activate script"
@@ -13,7 +22,7 @@ variable "cluster_name" {
 variable "cluster_image" {
     description = "single image for all cluster nodes - a convenience for CI"
     type = string
-    default = "openhpc-240116-1156-aa8dba7d" # https://github.com/stackhpc/ansible-slurm-appliance/pull/351
+    default = "openhpc-240116-1604-b3563a08" # https://github.com/stackhpc/ansible-slurm-appliance/pull/344
     # default = "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2"
 }
 

environments/common/inventory/group_vars/all/manila.yml

@@ -0,0 +1,13 @@
+# Default configuration for manila file shares, see
+# https://github.com/stackhpc/ansible-role-os-manila-mount
+# for all variable definitions, and override in your environment.
+
+os_manila_mount_shares: []
+  # - share_name:
+  #   share_user:
+  #   mount_path:
+  #   mount_user:
+  #   mount_group:
+  #   mount_mode:
+
+# os_manila_mount_ceph_version: nautilus # role default for RockyLinux 8

environments/common/layouts/everything

@@ -66,3 +66,6 @@ openhpc
 
 [proxy]
 # Hosts to configure http/s proxies - see ansible/roles/proxy/README.md
+
+[manila]
+# Hosts to configure for manila fileshares

environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf

@@ -1,5 +1,6 @@
 locals {
   user_data_path = "${var.environment_root}/cloud_init/${var.cluster_name}-%s.userdata.yml"
+  control_volumes = concat([openstack_blockstorage_volume_v3.state], var.home_volume_size > 0 ? [openstack_blockstorage_volume_v3.home][0] : [])
 }
 
 
@@ -100,20 +101,14 @@ resource "openstack_compute_instance_v2" "control" {
       delete_on_termination = true
   }
 
-  # state volume:
-  block_device {
-      destination_type = "volume"
-      source_type  = "volume"
-      boot_index = -1
-      uuid = openstack_blockstorage_volume_v3.state.id
-  }
-
-  # home volume:
-  block_device {
+  dynamic "block_device" {
+    for_each = local.control_volumes
+    content {
       destination_type = "volume"
       source_type  = "volume"
       boot_index = -1
-      uuid = openstack_blockstorage_volume_v3.home.id
+      uuid = block_device.value.id # actually openstack_blockstorage_volume_v3 id
+    }
   }
 
   network {
@@ -130,13 +125,15 @@ resource "openstack_compute_instance_v2" "control" {
     fqdn: ${var.cluster_name}-${each.key}.${var.cluster_name}.${var.cluster_domain_suffix}
 
     bootcmd:
-      %{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
+      %{for volume in local.control_volumes}
       - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
       %{endfor}
 
     mounts:
       - [LABEL=state, ${var.state_dir}]
+      %{if var.home_volume_size > 0}
       - [LABEL=home, /exports/home]
+      %{endif}
   EOF
 
 }

environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf

@@ -70,7 +70,7 @@ variable "state_volume_size" {
 variable "home_volume_size" {
     type = number
     description = "Size of state volume on control node, in GB"
-    default = 100 # GB
+    default = 100 # GB, 0 means no home volume
 }
 
 variable "vnic_type" {

environments/skeleton/{{cookiecutter.environment}}/terraform/volumes.tf

@@ -5,6 +5,9 @@ resource "openstack_blockstorage_volume_v3" "state" {
 }
 
 resource "openstack_blockstorage_volume_v3" "home" {
+
+    count = var.home_volume_size > 0 ? 1 : 0
+
     name = "${var.cluster_name}-home"
     description = "Home for control node" # first word used to label filesystem
     size = var.home_volume_size

requirements.txt

@@ -1,6 +1,7 @@
 ansible==6.0.0
 openstacksdk
 python-openstackclient
+python-manilaclient
 jmespath
 passlib[bcrypt]==1.7.4
 cookiecutter