WIP: sparse loop dev provisioning with systemd

buildchain/buildchain/salt_tree.py

@@ -651,7 +651,8 @@ def _get_parts(self) -> Iterator[str]:
     Path('salt/metalk8s/volumes/init.sls'),
     Path('salt/metalk8s/volumes/prepared/init.sls'),
     Path('salt/metalk8s/volumes/prepared/installed.sls'),
-    Path('salt/metalk8s/volumes/provisioned/init.sls'),
+    Path('salt/metalk8s/volumes/prepared/files/[email protected]'),
+    Path('salt/metalk8s/volumes/prepared/files/metalk8s-volume-clean-up'),
     Path('salt/metalk8s/volumes/unprepared/init.sls'),
 
     Path('salt/_auth/kubernetes_rbac.py'),

salt/_modules/metalk8s_volumes.py

@@ -62,39 +62,6 @@ def create(name):
     _get_volume(name).create()
 
 
-def is_provisioned(name):
-    """Check if the backing storage device is provisioned for the given volume.
-
-    Args:
-        name (str): volume name
-
-    Returns:
-        bool: True if the backing storage device is provisioned, otherwise False
-
-    CLI Example:
-
-    .. code-block:: bash
-
-        salt '<NODE_NAME>' metalk8s_volumes.is_provisioned example-volume
-    """
-    return _get_volume(name).is_provisioned
-
-
-def provision(name):
-    """Provision the backing storage device of the given volume.
-
-    Args:
-        name (str): volume name
-
-    CLI Example:
-
-    .. code-block:: bash
-
-        salt '<NODE_NAME>' metalk8s_volumes.provision example-volume
-    """
-    _get_volume(name).provision()
-
-
 def is_prepared(name):
     """Check if the given volume is prepared.
 
@@ -233,16 +200,6 @@ def create(self):  # pragma: no cover
         """Create the backing storage device."""
         return
 
-    @abc.abstractproperty
-    def is_provisioned(self):  # pragma: no cover
-        """Check if the backing storage device is provisioned."""
-        return
-
-    @abc.abstractmethod
-    def provision(self):  # pragma: no cover
-        """Provision the backing storage device."""
-        return
-
     @abc.abstractproperty
     def is_cleaned_up(self):  # pragma: no cover
         """Check if the backing storage device is cleaned up."""
@@ -347,32 +304,16 @@ def create(self):
                 self.path, exn
             ))
 
-    @property
-    def is_provisioned(self):
-        # A sparse loop device is provisioned when a sparse file is associated
-        # to a loop device.
-        command = ' '.join(['losetup', '--associated', self.path])
-        pattern = r'\({}\)'.format(re.escape(self.path))
-        result  = _run_cmd(command)
-        return re.search(pattern, result['stdout']) is not None
-
-    def provision(self):
-        command = ' '.join(list(self.PROVISIONING_COMMAND) + [self.path])
-        return _run_cmd(command)
-
     def prepare(self, force=False):
         # We format a "normal" file, not a block device: we need force=True.
         super(SparseLoopDevice, self).prepare(force=True)
 
     @property
     def is_cleaned_up(self):
-        return not (self.is_provisioned or self.exists)
+        return not self.exists
 
     def clean_up(self):
-        LOOP_CLR_FD = 0x4C01  # From /usr/include/linux/loop.h
         try:
-            with _open_fd(self.persistent_path, os.O_RDONLY) as fd:
-                fcntl.ioctl(fd, LOOP_CLR_FD, 0)
             os.remove(self.path)
         except OSError as exn:
             if exn.errno != errno.ENOENT:
@@ -420,13 +361,6 @@ def create(self):
             self.path
         ))
 
-    @property
-    def is_provisioned(self):
-        return True  # Nothing to do so it's always True.
-
-    def provision(self):
-        return  # Nothing to do
-
     @property
     def path(self):
         return self.get('spec.rawBlockDevice.devicePath')

salt/_states/metalk8s_volumes.py

@@ -51,43 +51,6 @@ def present(name):
     return ret
 
 
-def provisioned(name):
-    """Provision the given volume.
-
-    Args:
-        name (str): Volume name
-
-    Returns:
-        dict: state return value
-    """
-    ret = {'name': name, 'changes': {}, 'result': False, 'comment': ''}
-    # Idempotence.
-    if __salt__['metalk8s_volumes.is_provisioned'](name):
-        ret['result'] = True
-        ret['comment'] = 'Storage for volume {} already provisioned.'\
-            .format(name)
-        return ret
-    # Dry-run.
-    if __opts__['test']:
-        ret['changes'][name] = 'Provisioned'
-        ret['result'] = None
-        ret['comment'] = 'Storage for volume {} is going to be provisioned.'\
-            .format(name)
-        return ret
-    # Let's go for real.
-    try:
-        __salt__['metalk8s_volumes.provision'](name)
-    except CommandExecutionError as exn:
-        ret['result'] = False
-        ret['comment'] = 'Storage provisioning for volume {} failed: {}.'\
-            .format(name, exn)
-    else:
-        ret['changes'][name] = 'Provisioned'
-        ret['result'] = True
-        ret['comment'] = 'Storage provisioned for volume {}.'.format(name)
-    return ret
-
-
 def prepared(name):
     """Prepare the given volume.
 

salt/metalk8s/salt/minion/files/minion-99-metalk8s.conf.j2

@@ -10,8 +10,3 @@ use_superseded:
 log_level_logfile: {{ 'debug' if debug else 'info' }}
 
 saltenv: {{ saltenv }}
-
-# Prepare volume at startup (required for loop devices persistency).
-startup_states: sls
-sls_list:
-  - /metalk8s/volumes/provisioned

salt/metalk8s/volumes/prepared/files/metalk8s-sparse-volume-cleanup

@@ -0,0 +1,128 @@
+#!/usr/bin/env python2
+
+from __future__ import print_function
+import argparse
+import errno
+import os
+import stat
+import sys
+
+
+SPARSE_FILES_DIR = '/var/lib/metalk8s/storage/sparse/'
+LOOP_MAJOR = 7
+
+
+def parse_args(args=None):
+    parser = argparse.ArgumentParser(
+        prog='metalk8s-sparse-volume-cleanup',
+        help='Cleanup MetalK8s sparse loop volumes',
+    )
+
+    parser.add_argument(
+        '-i', '--uid',
+        required=True,
+        help='UID of the MetalK8s Volume object',
+        dest='volume_id',
+    )
+
+    return parser.parse_args(args=args)
+
+
+def check_sparse_file(volume_id):
+    """Check if the sparse file exists for this volume ID."""
+    sparse_file_path = SPARSE_FILES_DIR + volume_id
+    if not os.path.isfile(sparse_file_path):
+        die(
+            "Sparse file {} does not exist for volume '{}'".format(
+                sparse_file_path, volume_id
+            ),
+            exit_code=errno.ENOENT
+        )
+
+
+def cleanup(volume_id):
+    device_path = find_device(volume_id)
+
+    print("Detaching loop device '{}' for volume '{}'".format(
+        device_path, volume_id
+    ))
+
+    fd = os.open(device_path, os.O_RDONLY)
+    try:
+        fcntl.ioctl(fd, LOOP_CLR_FD, 0)
+    except IOError as exn:
+        if exn.errno != errno.ENXIO:
+            die("Unexpected error when trying to free device {}: {}".format(
+                device_path, str(exn)
+            ))
+        print("Device already freed", file=sys.stderr)
+    finally:
+        fd.close()
+
+    print("Loop device for volume '{}' was successfully detached".format(
+        volume_id
+    ))
+
+
+def main():
+    args = parse_args()
+
+    check_sparse_file(args.volume_id)
+    cleanup(args.volume_id)
+
+
+# Helpers {{{
+def die(message, exit_code=1):
+    print(message, file=sys.stderr)
+    sys.exit(exit_code)
+
+
+def find_device(volume_id):
+    """Find the device provisioned for this volume ID.
+    
+    SparseLoopDevice volumes are either:
+    - formatted, with the volume ID set in the filesystem annotations,
+      hence discoverable under /dev/disk/by-uuid/
+    - raw, in which case the device is partitioned and the first partition
+      is labeled with the volume ID, hence discoverable under
+      /dev/disk/by-partuuid/
+    """
+    path_by_uuid = "/dev/disk/by-uuid/{}".format(volume_id)
+    if os.path.exists(path_by_uuid):
+        device_path = os.path.realpath(path_by_uuid)
+    else:
+        path_by_partuuid = "/dev/disk/by-partuuid/{}".format(volume_id)
+        if not os.path.exists(path_by_partuuid):
+            die(
+                "Device for volume '{}' was not found".format(volume_id),
+                exit_code=errno.ENOENT,
+            )
+
+        partition_name = os.path.basename(os.path.realpath(path_by_partuuid))
+        device_name = os.path.basename(
+            os.path.realpath("/sys/class/block/{}/..".format(partition_name))
+        )
+        device_path = '/dev/{}'.format(match.group(0))
+
+    if not is_loop_device(device_path):
+        die("{} (found for volume '{}') is not a loop device".format(
+            device_path, volume_id
+        ))
+
+    return device_path
+
+
+def is_loop_device(path):
+    device_stat = os.stat(path)
+    return stat.S_ISBLK(device_stat.st_mode) \
+        and (_major(device_stat.st_rdev) == LOOP_MAJOR)
+
+
+def _major(value):
+    return (value >> 8) & 0xff
+
+
+# }}}
+
+if __name__ == '__main__':
+    main()

salt/metalk8s/volumes/prepared/files/metalk8s-sparse-volume@.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Setup MetalK8s sparse loop volume %I
+RequiresMountsFor=/var/lib/metalk8s/storage/sparse
+AssertFileNotEmpty=/var/lib/metalk8s/storage/sparse/%i
+
+[Service]
+Type=oneshot
+ExecStart=/bin/flock --exclusive --wait 10 /var/lock/metalk8s-sparse-volume.lock \
+             -c "/sbin/losetup --find --partscan /var/lib/metalk8s/storage/sparse/%i"
+ExecStop=/usr/local/bin/libexec/metalk8s-sparse-volume-cleanup --uid "%i"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target