From 87905aa5cbcf160838658eaf1e72d676a61cf4cb Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Tue, 23 Jul 2024 19:47:38 +0200
Subject: [PATCH 01/13] fc-luks: `test-open` subcommand for testing unlock

For streamlining the regular task of verifying whether (all) volumes of
a host can still be opened with the admin key and device key, I
implemented the `fc-luks keystore test-open` subcommand.

As this relies on the same volume discovery logic as the `reencrypt`
command, I refactored and reused that discoverly logic out into a
separate method.

Also includes a NixOS integration test for this, but no unit tests.
---
 pkgs/fc/ceph/src/fc/ceph/luks/manage.py     | 163 +++++++++++++++-----
 pkgs/fc/ceph/src/fc/ceph/main.py            |  17 +-
 pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py |  11 +-
 tests/ceph-nautilus.nix                     |  15 ++
 4 files changed, 160 insertions(+), 46 deletions(-)

diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
index ebbd7d956..f1db66797 100644
--- a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
+++ b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
@@ -5,9 +5,11 @@
 import secrets
 import shutil
 from pathlib import Path
+from subprocess import CalledProcessError
 from typing import NamedTuple, Optional
 
 from fc.ceph.luks import KEYSTORE  # singleton
+from fc.ceph.luks import Cryptsetup
 from fc.ceph.lvm import XFSVolume
 from fc.ceph.util import console, run
 
@@ -17,21 +19,59 @@ class LuksDevice(NamedTuple):
     name: str  # the LUKS name of the device
     # required for external header discovery, which we only utilise for backy
     mountpoint: Optional[str]
+    header: Optional[str] = None
 
+    @classmethod
+    def lsblk_to_cryptdevices(cls, lsblk_blockdevs: list) -> list["LuksDevice"]:
+        """parses the output of lsblk -Js -o NAME,PATH,TYPE,MOUNTPOINT"""
+        return [
+            cls(
+                base_blockdev=dev["children"][0]["path"],
+                name=dev["name"],
+                mountpoint=dev["mountpoint"],
+            )
+            for dev in lsblk_blockdevs
+            if dev["type"] == "crypt"
+        ]
 
-# TODO: better typing signature
-# Todo: should this be a static class method instead?
-def lsblk_to_cryptdevices(lsblk_blockdevs: list) -> list:
-    """parses the output of lsblk -Js -o NAME,PATH,TYPE,MOUNTPOINT"""
-    return [
-        LuksDevice(
-            base_blockdev=dev["children"][0]["path"],
-            name=dev["name"],
-            mountpoint=dev["mountpoint"],
+    @classmethod
+    def filter_cryptvolumes(
+        cls, name_glob: str, header: Optional[str]
+    ) -> list["LuksDevice"]:
+        """Retrieves visible crypt volumes via `lsblk`, filters their name to
+        match `name_glob`.
+
+        Optionally takes a path to an external `header` file, otherwise does
+        auto-discovery based on looking for a corresponding header file named
+        <mountpoint>.luks and passes an Optional[str].
+        """
+        candidates = cls.lsblk_to_cryptdevices(
+            run.json.lsblk("-s", "-o", "NAME,PATH,TYPE,MOUNTPOINT")
         )
-        for dev in lsblk_blockdevs
-        if dev["type"] == "crypt"
-    ]
+
+        matching_devs = []
+        for candidate in candidates:
+            if not fnmatch.fnmatch(candidate.name, name_glob):
+                continue
+
+            # adjust headers with autodetected heuristics
+            if (
+                (not header)
+                and (mp := candidate.mountpoint)
+                and os.path.exists(headerfile := f"{mp}.luks")
+            ):
+                matching_devs.append(candidate._replace(header=headerfile))
+            else:
+                matching_devs.append(candidate._replace(header=header))
+
+        if header and (match_count := len(matching_devs)) > 1:
+            raise ValueError(
+                f"Got {match_count} matching devices for glob '{name_glob}'.\n"
+                "When specifying an external header file, the target device "
+                "needs to be a single specific match."
+            )
+
+        return matching_devs
 
 
 class LUKSKeyStoreManager(object):
@@ -112,35 +152,9 @@ def rekey(
         else:
             raise ValueError(f"slot={slot}")
 
-        candidates = lsblk_to_cryptdevices(
-            run.json.lsblk("-s", "-o", "NAME,PATH,TYPE,MOUNTPOINT")
-        )
-        matching_devs = [
-            candidate
-            for candidate in candidates
-            if fnmatch.fnmatch(candidate.name, name_glob)
-        ]
-        if header and (match_count := len(matching_devs)) > 1:
-            raise ValueError(
-                f"Got {match_count} matching devices for glob '{name_glob}'.\n"
-                "When specifying an external header file, the target device "
-                "needs to be a single specific match."
-            )
-        for dev in matching_devs:
-            dev_header = header
-            console.print(f"Replacing key for {dev.name}")
-
-            if (
-                (not dev_header)
-                and (mp := dev.mountpoint)
-                and os.path.exists(headerfile := f"{mp}.luks")
-            ):
-                dev_header = headerfile
-            self._do_rekey(
-                slot=slot,
-                device=dev.base_blockdev,
-                header=dev_header,
-            )
+        for dev in LuksDevice.filter_cryptvolumes(name_glob, header=header):
+            console.print(f"Rekeying {dev.name}")
+            self._do_rekey(slot, device=dev.base_blockdev, header=dev.header)
 
         console.print("Key updated.", style="bold green")
 
@@ -179,6 +193,73 @@ def _do_rekey(self, slot: str, device: str, header: Optional[str]):
             input=add_input,
         )
 
+    def test_open(self, name_glob: str, header: Optional[str]) -> int:
+        # Ensure to request the admin key early on.
+        self._KEYSTORE.admin_key_for_input()
+
+        devices = LuksDevice.filter_cryptvolumes(name_glob, header=header)
+        if not devices:
+            console.print(
+                f"Warning: The glob `{name_glob}` matches no volume.",
+                style="yellow",
+            )
+            return 1
+
+        failing_devices = []
+        for dev in devices:
+            console.print(f"Test opening {dev.name}")
+            if not self._do_test_open(dev.base_blockdev, header=dev.header):
+                failing_devices.append(dev)
+
+        if failing_devices:
+            console.print(
+                "The following devices failed to open:\n"
+                + (
+                    "\n".join(
+                        (
+                            f"{dev.base_blockdev} ({dev.name})"
+                            for dev in failing_devices
+                        )
+                    )
+                ),
+                style="red",
+            )
+            return 2
+
+        return 0
+
+    def _do_test_open(self, device: str, header: Optional[str]) -> bool:
+        header_arg = [f"--header={header}"] if header else []
+        success = True
+
+        # test unlocking both with local key file as well as with admin key
+        try:
+            test_admin = Cryptsetup.cryptsetup(
+                "open",
+                "--test-passphrase",
+                device,
+                input=self._KEYSTORE.admin_key_for_input(),
+            )
+        except CalledProcessError:
+            console.print(
+                f"Failed to open {device} with admin passphrase.", style="red"
+            )
+            success = False
+        try:
+            test_local = Cryptsetup.cryptsetup(
+                "open",
+                "--test-passphrase",
+                f"--key-file={self._KEYSTORE.local_key_path()}",
+                device,
+            )
+        except CalledProcessError:
+            console.print(
+                f"Failed to open {device} with local key file.", style="red"
+            )
+            success = False
+
+        return success
+
         if header:
             self._KEYSTORE.backup_external_header(Path(header))
 
diff --git a/pkgs/fc/ceph/src/fc/ceph/main.py b/pkgs/fc/ceph/src/fc/ceph/main.py
index 1370eb837..f64e9d59c 100644
--- a/pkgs/fc/ceph/src/fc/ceph/main.py
+++ b/pkgs/fc/ceph/src/fc/ceph/main.py
@@ -452,7 +452,7 @@ def luks(args=sys.argv[1:]):
     parser_rekey = keystore_sub.add_parser("rekey", help="Rekey volumes.")
     parser_rekey.add_argument(
         "name_glob",
-        help="Names of LUKS volumes to update (globbing allowed), e.g. '*osd-*', 'backy'. Mutually exclusive with `--device`.",
+        help="Names of LUKS volumes to update (globbing allowed), e.g. '*osd-*', 'backy'.",
     )
     parser_rekey.add_argument(
         "--header",
@@ -467,6 +467,21 @@ def luks(args=sys.argv[1:]):
     )
     parser_rekey.set_defaults(action="rekey")
 
+    parser_test = keystore_sub.add_parser(
+        "test-open",
+        help="Test that encrypted volumes can be successfully unlocked.",
+    )
+    parser_test.add_argument(
+        "name_glob",
+        help="Names of LUKS volumes to check (globbing allowed), e.g. '*osd-*', 'backy'.",
+    )
+    parser_test.add_argument(
+        "--header",
+        help="When using an external LUKS header file, provide a path to it here."
+        "\nDefaults to autodetecting and using a file called ${mountpoint}.luks",
+    )
+    parser_test.set_defaults(action="test_open")
+
     parser_fingerprint = keystore_sub.add_parser(
         "fingerprint", help="Compute a fingerprint of an admin passphrase."
     )
diff --git a/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
index 94f1fe73c..8a86f2889 100644
--- a/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
+++ b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
@@ -327,36 +327,39 @@ def test_lsblk_to_cryptdevices():
     from fc.ceph.luks import manage
 
     assert set(
-        manage.lsblk_to_cryptdevices(json.loads(LSBLK_PATTY_JSON))
+        manage.LuksDevice.lsblk_to_cryptdevices(json.loads(LSBLK_PATTY_JSON))
     ) == set(
         (
             manage.LuksDevice(
                 base_blockdev="/dev/mapper/vgsys-ceph--mon--crypted",
                 name="ceph-mon",
                 mountpoint="/srv/ceph/mon/ceph-patty",
+                header=None,
             ),
             manage.LuksDevice(
                 base_blockdev="/dev/sdb1",
                 name="backy",
                 mountpoint="/srv/backy",
+                header=None,
             ),
             manage.LuksDevice(
                 base_blockdev="/dev/mapper/vgsys-ceph--mgr--crypted",
                 name="ceph-mgr",
                 mountpoint="/srv/ceph/mgr/ceph-patty",
+                header=None,
             ),
         )
     )
-    assert manage.lsblk_to_cryptdevices([]) == []
+    assert manage.LuksDevice.lsblk_to_cryptdevices([]) == []
     # disks, but no crypt and no children
     assert (
-        manage.lsblk_to_cryptdevices(
+        manage.LuksDevice.lsblk_to_cryptdevices(
             [{"name": "sdb", "path": "/dev/sda", "type": "disk"}]
         )
         == []
     )
     # disks with mountpoint = null
-    assert manage.lsblk_to_cryptdevices(
+    assert manage.LuksDevice.lsblk_to_cryptdevices(
         [
             {
                 "name": "ceph-osd3-block",
diff --git a/tests/ceph-nautilus.nix b/tests/ceph-nautilus.nix
index af95dbd58..886289cd8 100644
--- a/tests/ceph-nautilus.nix
+++ b/tests/ceph-nautilus.nix
@@ -397,6 +397,8 @@ in
       show(host1, "vgs")
       assert_clean_cluster(host2, 3, 3, 3, 320)
 
+    # TODO: this is non-ceph-specific and could be moved to a separate test.
+    # But we need some encrypted volumes to work with.
     with subtest("Destroy and re-create the keystore, rekey the OSD"):
       host1.succeed("fc-luks keystore destroy --no-overwrite > /dev/kmsg 2>&1")
       host1.succeed("fc-luks keystore create /dev/vde > /dev/kmsg 2>&1")
@@ -434,6 +436,19 @@ in
       host3.succeed('fc-ceph osd create-bluestore --no-encrypt --wal=external /dev/vdc > /dev/kmsg 2>&1')
       assert_clean_cluster(host2, 3, 3, 3, 320)
 
+    with subtest("`fc-luks keystore test-open` verifies that volumes can be unlocked"):
+      host1.succeed('echo -e "newphrase\n" | setsid -w fc-luks keystore test-open "*" > /dev/kmsg 2>&1')
+      host1.succeed('echo -e "newphrase\n" | setsid -w fc-luks keystore test-open "*osd-*" > /dev/kmsg 2>&1')
+      # no volume matched
+      host1.fail('echo -e "newphrase\n" | setsid -w fc-luks keystore test-open "asdfhjkl" > /dev/kmsg 2>&1')
+      # wrong admin passphrase (will also modify the fingerprint)
+      host1.fail('echo -e "wrongphrase\ny\n" | setsid -w fc-luks keystore test-open "*" > /dev/kmsg 2>&1')
+      # wrong local keyfile (will correct admin keyphrase fingerprint again)
+      host1.execute("mv /mnt/keys/host1.key{,.bak}; echo garbage > /mnt/keys/host1.key")
+      host1.fail('echo -e "newphrase\ny\n" | setsid -w fc-luks keystore test-open "*" > /dev/kmsg 2>&1')
+      host1.execute("mv /mnt/keys/host1.key{.bak,}")
+
+    # FIXME: deleting this might affect following tests, especially the reboot test
     with subtest("The check discovers non-conforming host key conditions"):
       host1.execute('chmod o+rw /mnt/keys/*')
       host1.fail("${check_key_file_cmd} > /dev/kmsg 2>&1")

From 86f82da2ffc67fa79223d1ac46567a8a4345a073 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Mon, 29 Jul 2024 18:26:32 +0200
Subject: [PATCH 02/13] tests.ceph: fix and activate test for successful bootup

Reactivating the already-implemented, but commented-out test for
successful daemon activation after reboots of encrypted ceph hosts.
The test was broken due to stateful modifications of the key material in
previous parts of the test, moving the subtest in front of these
modifications resolved it.

PL-132687
---
 tests/ceph-nautilus.nix | 56 ++++++++++++++++++-----------------------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/tests/ceph-nautilus.nix b/tests/ceph-nautilus.nix
index 886289cd8..bdcfdac77 100644
--- a/tests/ceph-nautilus.nix
+++ b/tests/ceph-nautilus.nix
@@ -436,6 +436,31 @@ in
       host3.succeed('fc-ceph osd create-bluestore --no-encrypt --wal=external /dev/vdc > /dev/kmsg 2>&1')
       assert_clean_cluster(host2, 3, 3, 3, 320)
 
+    with subtest("Verify keystore automount on boot"):
+      host1.execute("systemctl poweroff --force")
+      host1.wait_for_shutdown()
+      host1.start()
+      host1.wait_for_unit("local-fs.target")
+      show(host1, "mount")
+      host1.execute("sleep 5")
+      show(host1, "mount")
+      show(host1, "systemctl status multi-user.target")
+      show(host1, "lsblk")
+      show(host1, "cat /etc/fstab")
+      host1.succeed("${pkgs.util-linux}/bin/findmnt /mnt/keys > /dev/kmsg 2>&1")
+
+    with subtest("Verify all services are up after a reboot"):
+      #host1.sleep(30)
+      show(host1, "systemctl status -l fc-ceph-mon.service")
+      show(host1, "systemctl status -l fc-ceph-mgr.service")
+      show(host1, "systemctl status -l fc-ceph-rgw.service")
+      show(host1, "systemctl status -l fc-ceph-osd@0.service")
+      host1.wait_for_unit("fc-ceph-mon.service")
+      host1.wait_for_unit("fc-ceph-mgr.service")
+      host1.wait_for_unit("fc-ceph-rgw.service")
+      host1.wait_for_unit("fc-ceph-osds-all.service")
+      host1.wait_for_unit("fc-ceph-osd@0.service")
+
     with subtest("`fc-luks keystore test-open` verifies that volumes can be unlocked"):
       host1.succeed('echo -e "newphrase\n" | setsid -w fc-luks keystore test-open "*" > /dev/kmsg 2>&1')
       host1.succeed('echo -e "newphrase\n" | setsid -w fc-luks keystore test-open "*osd-*" > /dev/kmsg 2>&1')
@@ -502,37 +527,6 @@ in
     #  print(snapfillcheck[1])
     #  assert snapfillcheck[0] == 2
 
-    with subtest("Verify keystore automount on boot"):
-      host1.execute("systemctl poweroff --force")
-      host1.wait_for_shutdown()
-      host1.start()
-      host1.wait_for_unit("local-fs.target")
-      show(host1, "mount")
-      host1.execute("sleep 5")
-      show(host1, "mount")
-      show(host1, "systemctl status multi-user.target")
-      show(host1, "lsblk")
-      show(host1, "cat /etc/fstab")
-      host1.succeed("${pkgs.util-linux}/bin/findmnt /mnt/keys > /dev/kmsg 2>&1")
-
-    #with subtest("Verify all services are up after a reboot"):
-    #  host1.sleep(30)
-    #  show(host1, "systemctl status -l fc-ceph-mon.service")
-    #  show(host1, "cat /var/log/ceph/ceph-mon.host1.log")
-    #  show(host1, "systemctl status -l fc-ceph-mgr.service")
-    #  show(host1, "cat /var/log/ceph/ceph-mgr.host1.log")
-    #  show(host1, "systemctl status -l fc-ceph-rgw.service")
-    #  show(host1, "systemctl status -l fc-ceph-osd@0.service")
-    #  show(host1, "cat /var/log/ceph/ceph-osd.0.log")
-    #  show(host1, "journalctl -b -u systemd-tmpfiles-setup.service")
-    #  show(host1, "stat /run")
-    #  show(host1, "stat /run/ceph")
-    #  host1.wait_for_unit("fc-ceph-mon.service")
-    #  host1.wait_for_unit("fc-ceph-mgr.service")
-    #  host1.wait_for_unit("fc-ceph-rgw.service")
-    #  host1.wait_for_unit("fc-ceph-osds-all.service")
-    #  host1.wait_for_unit("fc-ceph-osd@0.service")
-
     print("Time spent waiting", time_waiting)
   '';
 })

From 5a4f3bb21d9962c98ff692f075d29103d4707767 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Wed, 31 Jul 2024 12:12:15 +0200
Subject: [PATCH 03/13] remove unused deprecated python27-ceph-downgrades

This has been unused since we dropped Ceph Jewel.

Piggybacking this in this branch as FDE work is slightly related to
storage and ceph.
---
 pkgs/overlay.nix | 30 ------------------------------
 1 file changed, 30 deletions(-)

diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix
index 4babc386c..26e032ca1 100644
--- a/pkgs/overlay.nix
+++ b/pkgs/overlay.nix
@@ -528,36 +528,6 @@ in {
   prometheus-elasticsearch-exporter = super.callPackage ./prometheus-elasticsearch-exporter.nix { };
 
   py_pytest_patterns = self.callPackage ./python/pytest-patterns { };
-  # python27 with several downgrades to make required modules work under python27 again
-  python27-ceph-downgrades = let thisPy = self.python27-ceph-downgrades;
-  in
-  super.python27.override {
-    packageOverrides = python-self: python-super: {
-      cheroot = thisPy.pkgs.callPackage ./python/cheroot { };
-      cherrypy = thisPy.pkgs.callPackage ./python/cherrypy { };
-      cython = thisPy.pkgs.callPackage ./python/Cython { };
-      jaraco_text = thisPy.pkgs.callPackage ./python/jaraco_text { };
-      PasteDeploy = python-super.PasteDeploy.overrideAttrs (oldattrs: {
-        # for pkg_resources
-        propagatedBuildInputs = oldattrs.propagatedBuildInputs ++ [python-self.setuptools];
-      });
-      pecan = thisPy.pkgs.callPackage ./python/pecan { };
-      portend = thisPy.pkgs.callPackage ./python/portend { };
-      pypytools = thisPy.pkgs.callPackage ./python/pypytools { };
-      pyquery = thisPy.pkgs.callPackage ./python/pyquery { };
-      routes = python-super.routes.overrideAttrs (oldattrs: {
-        # work around a weird pythonImportsCheck failure, but cannot be empty
-        pythonImportsCheckPhase = "true";
-      });
-      tempora = thisPy.pkgs.callPackage ./python/tempora { };
-      waitress = thisPy.pkgs.callPackage ./python/waitress { };
-      webtest = thisPy.pkgs.callPackage ./python/webtest {
-        pastedeploy = python-self.PasteDeploy;
-      };
-      WebTest = python-self.webtest;
-      zc_lockfile = thisPy.pkgs.callPackage ./python/zc_lockfile { };
-    };
-  };
 
   # Speed up NixOS tests by making the 9p file system more efficient.
   qemu = super.qemu.overrideAttrs (o: {

From 674c39f85181e70822b8d9e1638287f0ae9d1d33 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Thu, 1 Aug 2024 21:28:48 +0200
Subject: [PATCH 04/13] physical machines: sensu check that swap is off

When doing full-disk encryption, persisted encrypted data from disk can
still be kept in memory in a decrypted state. When swapping memory pages
to disk again, there is the danger of persisting that plaintext data to
disk again, circumventing the goal of the encryption.
As we do not want to use swap on physical machines anymore, we can just
monitor and warn when swap is unexpectedly used.

PL-131325
---
 .../infrastructure/flyingcircus-physical.nix  |  1 -
 nixos/platform/full-disk-encryption.nix       | 20 +++++++++++++++----
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/nixos/infrastructure/flyingcircus-physical.nix b/nixos/infrastructure/flyingcircus-physical.nix
index 8b9db5293..25106aa02 100644
--- a/nixos/infrastructure/flyingcircus-physical.nix
+++ b/nixos/infrastructure/flyingcircus-physical.nix
@@ -4,7 +4,6 @@ with lib;
 
 let
   cfg = config.flyingcircus;
-  inherit (config) fclib;
 in
 mkIf (cfg.infrastructureModule == "flyingcircus-physical") {
 
diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 29ed83e4d..1325e43a8 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -58,10 +58,22 @@ in
         cephPkgs.fc-ceph
       ];
 
-      flyingcircus.services.sensu-client.checks.keystickMounted = {
-        notification = "USB stick with disk encryption keys is mounted and keyfile is readable.";
-        interval = 60;
-        command = "sudo ${check_key_file}";
+      flyingcircus.services.sensu-client.checks = {
+        keystickMounted = {
+          notification = "USB stick with disk encryption keys is mounted and keyfile is readable.";
+          interval = 60;
+          command = "sudo ${check_key_file}";
+        };
+        noSwap = {
+          notification = "Machine does not use swap to arbitrarily persist memory pages with sensitive data.";
+          interval = 60;
+          command = toString (pkgs.writeShellScript "noSwapCheck" ''
+            # /proc/swaps always has a header line
+            if [ $(${pkgs.coreutils}/bin/cat /proc/swaps | ${pkgs.coreutils}/bin/wc -l) -ne 1 ]; then
+              exit 1
+            fi
+          '');
+        };
       };
 
       flyingcircus.passwordlessSudoRules = [{

From df8beaee0c72c5e00dfcbf5e5b564a1f547a0277 Mon Sep 17 00:00:00 2001
From: Christian Theune <ct@flyingcircus.io>
Date: Wed, 21 Aug 2024 15:28:01 +0200
Subject: [PATCH 05/13] implement check to monitor important parameters

---
 pkgs/fc/ceph/src/fc/ceph/luks/check.py | 152 +++++++++++++++++++++++++
 1 file changed, 152 insertions(+)
 create mode 100644 pkgs/fc/ceph/src/fc/ceph/luks/check.py

diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/check.py b/pkgs/fc/ceph/src/fc/ceph/luks/check.py
new file mode 100644
index 000000000..4df0348ad
--- /dev/null
+++ b/pkgs/fc/ceph/src/fc/ceph/luks/check.py
@@ -0,0 +1,152 @@
+import sys
+
+data = """
+LUKS header information
+Version:        2
+Epoch:          8
+Metadata area:  16384 [bytes]
+Keyslots area:  16744448 [bytes]
+UUID:           25a97553-456d-424d-b753-0d9f4f6f928f
+Label:          (no label)
+Subsystem:      (no subsystem)
+Flags:          (no flags)
+
+Data segments:
+  0: crypt
+    offset: 16777216 [bytes]
+    length: (whole device)
+    cipher: aes-xts-plain64
+    sector: 4096 [bytes]
+
+Keyslots:
+  0: luks2
+    Key:        512 bits
+    Priority:   normal
+    Cipher:     aes-xts-plain64
+    Cipher key: 512 bits
+    PBKDF:      argon2id
+    Time cost:  4
+    Memory:     992242
+    Threads:    4
+    Salt:       44 42 a3 41 fd be f7 5f 71 ed 79 1e e5 ec a8 d6
+                54 b0 71 04 57 db 8b 26 d5 ee 27 d7 0a e6 20 08
+    AF stripes: 4000
+    AF hash:    sha256
+    Area offset:32768 [bytes]
+    Area length:258048 [bytes]
+    Digest ID:  0
+  1: luks2
+    Key:        512 bits
+    Priority:   normal
+    Cipher:     aes-xts-plain64
+    Cipher key: 512 bits
+    PBKDF:      argon2id
+    Time cost:  4
+    Memory:     983424
+    Threads:    4
+    Salt:       f1 23 25 d0 49 fe 58 7d 2e c4 c2 03 b9 ad 3b 91
+                5e 6b 23 8e d6 2f d5 28 6e 0c 71 36 55 db 1c 8c
+    AF stripes: 4000
+    AF hash:    sha256
+    Area offset:290816 [bytes]
+    Area length:258048 [bytes]
+    Digest ID:  0
+Tokens:
+Digests:
+  0: pbkdf2
+    Hash:       sha256
+    Iterations: 77010
+    Salt:       a1 96 76 95 83 07 61 84 32 86 33 88 21 0d c6 b9
+                89 a3 f5 b6 19 a6 e6 07 6b 56 b2 0f 2a bd 3e 87
+    Digest:     c5 11 30 cb 22 f3 e6 31 c1 c7 0c 38 c1 b8 67 bd
+                34 75 6e 15 2d 26 30 7b e2 25 4a 66 62 9c ff e8
+"""
+
+
+# pbkdf: argon2id
+
+
+def check_cipher(lines):
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Cipher:"):
+            continue
+        cipher = line.split(":")[1].strip()
+        if cipher != "aes-xts-plain64":
+            yield f"cipher: {cipher} does not match aes-xts-plain64"
+
+
+def extract_keyslot_numbers(lines):
+    known_keyslots = set()
+    lines_iter = iter(lines)
+    for line in lines_iter:
+        if line.startswith("Keyslots:"):
+            break
+    else:
+        return known_keyslots
+
+    for line in lines_iter:
+        if line.startswith("Tokens:"):
+            break
+        if not ":" in line:
+            continue
+        header, value = line.split(":")
+        try:
+            header = int(header.strip())
+        except Exception:
+            continue
+        assert value.strip() == "luks2", (header, value, line)
+        known_keyslots.add(header)
+    return known_keyslots
+
+
+def check_key_slots_exactly_1_and_0(lines):
+    keyslots = extract_keyslot_numbers(lines)
+    if set([0, 1]) != keyslots:
+        yield f"keyslots: unexpected configuration ({keyslots})"
+
+
+def check_512_bit_keys(lines):
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Key:"):
+            continue
+        key_size = line.split(":")[1].strip()
+        if key_size != "512 bits":
+            yield f"keysize: {key_size} does not match expected 512 bits"
+
+
+def check_pbkdf_is_argon2id(lines):
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("PBKDF:"):
+            continue
+        pbkdf = line.split(":")[1].strip()
+        if pbkdf != "argon2id":
+            yield f"pbkdf: {pbkdf} does not match expected argon2id"
+
+
+def main():
+    checks = [
+        check_cipher,
+        check_key_slots_exactly_1_and_0,
+        check_512_bit_keys,
+        check_pbkdf_is_argon2id,
+    ]
+    lines = data.splitlines()
+    errors = 0
+    for check in checks:
+        check_ok = True
+        for error in check(lines):
+            errors += 1
+            check_ok = False
+            print(f"{check.__name__}: {error}")
+        if check_ok:
+            print(f"{check.__name__}: OK")
+
+    if errors:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

From b1c77fcda97cfb8f133f8d7c91b4fa1c709afbb7 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Wed, 21 Aug 2024 21:13:58 +0200
Subject: [PATCH 06/13] fc-luks check: establish sensu-check

Establish a regular sensu check on physical hosts that checks certain
LUKS header parameters of all encrypted devices, to discover unexpected
diversions.

- tie together the individual condition checks
- integrate checks into fc-luks tooling with discovery of encrypted
  volumes
- add sensu check
- extend checks with plausibility checks for proper dump input
- add unit and integration tests for checks with proper mocking
---
 nixos/platform/full-disk-encryption.nix       |   9 +-
 pkgs/fc/ceph/src/fc/ceph/luks/check.py        | 152 ---------
 pkgs/fc/ceph/src/fc/ceph/luks/checks.py       |  89 ++++++
 pkgs/fc/ceph/src/fc/ceph/luks/manage.py       |  30 ++
 pkgs/fc/ceph/src/fc/ceph/main.py              |  16 +
 .../ceph/src/fc/ceph/tests/test_luks_dump.py  | 300 ++++++++++++++++++
 6 files changed, 442 insertions(+), 154 deletions(-)
 delete mode 100644 pkgs/fc/ceph/src/fc/ceph/luks/check.py
 create mode 100644 pkgs/fc/ceph/src/fc/ceph/luks/checks.py
 create mode 100644 pkgs/fc/ceph/src/fc/ceph/tests/test_luks_dump.py

diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 1325e43a8..98f6fcb31 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -29,8 +29,8 @@ let
 
       exit 0
     '';
-
   cephPkgs = fclib.ceph.mkPkgs "nautilus";  # FIXME: just a workaround
+  check_luks_cmd = "${cephPkgs.fc-ceph}/bin/fc-luks check '*'";
 in
 {
 
@@ -74,10 +74,15 @@ in
             fi
           '');
         };
+        luksParams = {
+          notification = "LUKS Volumes use expected parameters.";
+          interval = 3600;
+          command = "sudo ${check_luks_cmd}";
+        };
       };
 
       flyingcircus.passwordlessSudoRules = [{
-        commands = [(toString check_key_file)];
+        commands = [(toString check_key_file) check_luks_cmd];
         groups = ["sensuclient"];
       }];
 
diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/check.py b/pkgs/fc/ceph/src/fc/ceph/luks/check.py
deleted file mode 100644
index 4df0348ad..000000000
--- a/pkgs/fc/ceph/src/fc/ceph/luks/check.py
+++ /dev/null
@@ -1,152 +0,0 @@
-import sys
-
-data = """
-LUKS header information
-Version:        2
-Epoch:          8
-Metadata area:  16384 [bytes]
-Keyslots area:  16744448 [bytes]
-UUID:           25a97553-456d-424d-b753-0d9f4f6f928f
-Label:          (no label)
-Subsystem:      (no subsystem)
-Flags:          (no flags)
-
-Data segments:
-  0: crypt
-    offset: 16777216 [bytes]
-    length: (whole device)
-    cipher: aes-xts-plain64
-    sector: 4096 [bytes]
-
-Keyslots:
-  0: luks2
-    Key:        512 bits
-    Priority:   normal
-    Cipher:     aes-xts-plain64
-    Cipher key: 512 bits
-    PBKDF:      argon2id
-    Time cost:  4
-    Memory:     992242
-    Threads:    4
-    Salt:       44 42 a3 41 fd be f7 5f 71 ed 79 1e e5 ec a8 d6
-                54 b0 71 04 57 db 8b 26 d5 ee 27 d7 0a e6 20 08
-    AF stripes: 4000
-    AF hash:    sha256
-    Area offset:32768 [bytes]
-    Area length:258048 [bytes]
-    Digest ID:  0
-  1: luks2
-    Key:        512 bits
-    Priority:   normal
-    Cipher:     aes-xts-plain64
-    Cipher key: 512 bits
-    PBKDF:      argon2id
-    Time cost:  4
-    Memory:     983424
-    Threads:    4
-    Salt:       f1 23 25 d0 49 fe 58 7d 2e c4 c2 03 b9 ad 3b 91
-                5e 6b 23 8e d6 2f d5 28 6e 0c 71 36 55 db 1c 8c
-    AF stripes: 4000
-    AF hash:    sha256
-    Area offset:290816 [bytes]
-    Area length:258048 [bytes]
-    Digest ID:  0
-Tokens:
-Digests:
-  0: pbkdf2
-    Hash:       sha256
-    Iterations: 77010
-    Salt:       a1 96 76 95 83 07 61 84 32 86 33 88 21 0d c6 b9
-                89 a3 f5 b6 19 a6 e6 07 6b 56 b2 0f 2a bd 3e 87
-    Digest:     c5 11 30 cb 22 f3 e6 31 c1 c7 0c 38 c1 b8 67 bd
-                34 75 6e 15 2d 26 30 7b e2 25 4a 66 62 9c ff e8
-"""
-
-
-# pbkdf: argon2id
-
-
-def check_cipher(lines):
-    for line in lines:
-        line = line.strip()
-        if not line.startswith("Cipher:"):
-            continue
-        cipher = line.split(":")[1].strip()
-        if cipher != "aes-xts-plain64":
-            yield f"cipher: {cipher} does not match aes-xts-plain64"
-
-
-def extract_keyslot_numbers(lines):
-    known_keyslots = set()
-    lines_iter = iter(lines)
-    for line in lines_iter:
-        if line.startswith("Keyslots:"):
-            break
-    else:
-        return known_keyslots
-
-    for line in lines_iter:
-        if line.startswith("Tokens:"):
-            break
-        if not ":" in line:
-            continue
-        header, value = line.split(":")
-        try:
-            header = int(header.strip())
-        except Exception:
-            continue
-        assert value.strip() == "luks2", (header, value, line)
-        known_keyslots.add(header)
-    return known_keyslots
-
-
-def check_key_slots_exactly_1_and_0(lines):
-    keyslots = extract_keyslot_numbers(lines)
-    if set([0, 1]) != keyslots:
-        yield f"keyslots: unexpected configuration ({keyslots})"
-
-
-def check_512_bit_keys(lines):
-    for line in lines:
-        line = line.strip()
-        if not line.startswith("Key:"):
-            continue
-        key_size = line.split(":")[1].strip()
-        if key_size != "512 bits":
-            yield f"keysize: {key_size} does not match expected 512 bits"
-
-
-def check_pbkdf_is_argon2id(lines):
-    for line in lines:
-        line = line.strip()
-        if not line.startswith("PBKDF:"):
-            continue
-        pbkdf = line.split(":")[1].strip()
-        if pbkdf != "argon2id":
-            yield f"pbkdf: {pbkdf} does not match expected argon2id"
-
-
-def main():
-    checks = [
-        check_cipher,
-        check_key_slots_exactly_1_and_0,
-        check_512_bit_keys,
-        check_pbkdf_is_argon2id,
-    ]
-    lines = data.splitlines()
-    errors = 0
-    for check in checks:
-        check_ok = True
-        for error in check(lines):
-            errors += 1
-            check_ok = False
-            print(f"{check.__name__}: {error}")
-        if check_ok:
-            print(f"{check.__name__}: OK")
-
-    if errors:
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/checks.py b/pkgs/fc/ceph/src/fc/ceph/luks/checks.py
new file mode 100644
index 000000000..7332faee3
--- /dev/null
+++ b/pkgs/fc/ceph/src/fc/ceph/luks/checks.py
@@ -0,0 +1,89 @@
+from typing import Iterator
+
+# In case these checks ever break: More recent cryptsetup versions also support
+# JSON-formatted output via `cryptsetup luksDump --dump-json-metadata <dev>`.
+# It most likely makes sense parsing that data instead once available.
+
+
+def check_cipher(lines: str) -> Iterator[str]:
+    checked = False  # plausibility check: did we get any cipher info?
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Cipher:"):
+            continue
+        cipher = line.split(":")[1].strip()
+        if cipher != "aes-xts-plain64":
+            yield f"cipher: {cipher} does not match aes-xts-plain64"
+        checked = True
+
+    if not checked:
+        yield "Unable to check cipher correctness, no `Cipher:` found in dump"
+
+
+def _extract_keyslot_numbers(lines: str):
+    known_keyslots: set[int] = set()
+    lines_iter = iter(lines)
+    for line in lines_iter:
+        if line.startswith("Keyslots:"):
+            break
+    else:
+        return known_keyslots
+
+    for line in lines_iter:
+        if line.startswith("Tokens:"):
+            break
+        if not ":" in line:
+            continue
+        header, value = line.split(":")
+        try:
+            header_i = int(header.strip())
+        except Exception:
+            continue
+        assert value.strip() == "luks2", (header, value, line)
+        known_keyslots.add(header_i)
+    return known_keyslots
+
+
+def check_key_slots_exactly_1_and_0(lines: str) -> Iterator[str]:
+    keyslots = _extract_keyslot_numbers(lines)
+    if set([0, 1]) != keyslots:
+        yield f"keyslots: unexpected configuration ({keyslots})"
+
+
+def check_512_bit_keys(lines: str) -> Iterator[str]:
+    checked = False
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Key:"):
+            continue
+        key_size = line.split(":")[1].strip()
+        if key_size != "512 bits":
+            yield f"keysize: {key_size} does not match expected 512 bits"
+        checked = True
+
+    if not checked:
+        yield "Unable to check key size correctness, no `Key:` found in dump"
+
+
+def check_pbkdf_is_argon2id(lines: str) -> Iterator[str]:
+    checked = False
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("PBKDF:"):
+            continue
+        pbkdf = line.split(":")[1].strip()
+        if pbkdf != "argon2id":
+            yield f"pbkdf: {pbkdf} does not match expected argon2id"
+        checked = True
+
+    if not checked:
+        yield "Unable to check PBKDF correctness, no `PBKDF:` found in dump"
+
+
+# All these checks work on a list of lines output by `cryptsetup luksDump`
+all_checks = [
+    check_cipher,
+    check_key_slots_exactly_1_and_0,
+    check_512_bit_keys,
+    check_pbkdf_is_argon2id,
+]
diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
index f1db66797..e735ed89b 100644
--- a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
+++ b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
@@ -10,6 +10,7 @@
 
 from fc.ceph.luks import KEYSTORE  # singleton
 from fc.ceph.luks import Cryptsetup
+from fc.ceph.luks.checks import all_checks
 from fc.ceph.lvm import XFSVolume
 from fc.ceph.util import console, run
 
@@ -193,6 +194,35 @@ def _do_rekey(self, slot: str, device: str, header: Optional[str]):
             input=add_input,
         )
 
+    @staticmethod
+    def check_luks(name_glob: str, header: Optional[str]) -> int:
+        devices = LuksDevice.filter_cryptvolumes(name_glob, header=header)
+        if not devices:
+            console.print(
+                f"Warning: The glob `{name_glob}` matches no volume.",
+                style="yellow",
+            )
+            return 1
+
+        errors = 0
+        for dev in devices:
+            console.print(f"Checking {dev.name}:")
+            dump_lines = (
+                Cryptsetup.cryptsetup("luksDump", dev.base_blockdev)
+                .decode("utf-8")
+                .splitlines()
+            )
+            for check in all_checks:
+                check_ok = True
+                for error in check(dump_lines):
+                    errors += 1
+                    check_ok = False
+                    console.print(f"{check.__name__}: {error}", style="red")
+                if check_ok:
+                    console.print(f"{check.__name__}: OK", style="green")
+
+        return 1 if errors else 0
+
     def test_open(self, name_glob: str, header: Optional[str]) -> int:
         # Ensure to request the admin key early on.
         self._KEYSTORE.admin_key_for_input()
diff --git a/pkgs/fc/ceph/src/fc/ceph/main.py b/pkgs/fc/ceph/src/fc/ceph/main.py
index f64e9d59c..069e7f0ea 100644
--- a/pkgs/fc/ceph/src/fc/ceph/main.py
+++ b/pkgs/fc/ceph/src/fc/ceph/main.py
@@ -425,6 +425,22 @@ def luks(args=sys.argv[1:]):
     # during construction of individual subcommands and might be re-used for
     # different command sections (mon, osd, ...)
 
+    parser_check = subparsers.add_parser(
+        "check", help="Check LUKS metadata header parameters."
+    )
+    parser_check.set_defaults(
+        subsystem=fc.ceph.luks.manage.LUKSKeyStoreManager, action="check_luks"
+    )
+    parser_check.add_argument(
+        "name_glob",
+        help="Names of LUKS volumes to check (globbing allowed), e.g. '*osd-*', 'backy'.",
+    )
+    parser_check.add_argument(
+        "--header",
+        help="When using an external LUKS header file, provide a path to it here."
+        "\nDefaults to autodetecting and using a file called ${mountpoint}.luks",
+    )
+
     keystore = subparsers.add_parser("keystore", help="Manage the keystore.")
     keystore.set_defaults(
         subsystem=fc.ceph.luks.manage.LUKSKeyStoreManager,
diff --git a/pkgs/fc/ceph/src/fc/ceph/tests/test_luks_dump.py b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks_dump.py
new file mode 100644
index 000000000..62d8ebd3d
--- /dev/null
+++ b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks_dump.py
@@ -0,0 +1,300 @@
+import textwrap
+import unittest.mock
+from unittest.mock import MagicMock
+
+data_correct = """LUKS header information
+Version:       	2
+Epoch:         	8
+Metadata area: 	16384 [bytes]
+Keyslots area: 	16744448 [bytes]
+UUID:          	3e2649b4-88c9-4b3f-acf5-edcc380ecc23
+Label:         	(no label)
+Subsystem:     	(no subsystem)
+Flags:       	(no flags)
+
+Data segments:
+  0: crypt
+	offset: 16777216 [bytes]
+	length: (whole device)
+	cipher: aes-xts-plain64
+	sector: 4096 [bytes]
+
+Keyslots:
+  0: luks2
+	Key:        512 bits
+	Priority:   normal
+	Cipher:     aes-xts-plain64
+	Cipher key: 512 bits
+	PBKDF:      argon2id
+	Time cost:  4
+	Memory:     1023865
+	Threads:    4
+	Salt:       ec e0 7a bf 23 d1 b0 13 ac 97 b3 8d 7b 6e 5c 4f
+	            21 a0 c1 19 f1 1e f7 0d 2b 67 3d e8 8a 4c 51 47
+	AF stripes: 4000
+	AF hash:    sha256
+	Area offset:32768 [bytes]
+	Area length:258048 [bytes]
+	Digest ID:  0
+  1: luks2
+	Key:        512 bits
+	Priority:   normal
+	Cipher:     aes-xts-plain64
+	Cipher key: 512 bits
+	PBKDF:      argon2id
+	Time cost:  4
+	Memory:     989547
+	Threads:    4
+	Salt:       7a 07 6e a9 20 ac 1e 8a 6b 87 4d 26 51 66 d1 ce
+	            f8 35 21 9a 2e 8d dc b6 9b d5 50 b8 f2 e1 11 3f
+	AF stripes: 4000
+	AF hash:    sha256
+	Area offset:290816 [bytes]
+	Area length:258048 [bytes]
+	Digest ID:  0
+Tokens:
+Digests:
+  0: pbkdf2
+	Hash:       sha256
+	Iterations: 75851
+	Salt:       3b 44 80 a2 6f 29 26 01 98 bb a2 92 cc 6e bc 7c
+	            df a9 e2 b2 90 ad 5e 4a e2 75 bf 9e ac 3f 81 59
+	Digest:     76 11 6a c0 53 28 aa 6c 89 a9 24 52 7a d9 51 39
+	            b4 d9 0e 91 50 2c 5a d4 ab df a2 6a 98 8f b1 ed"""
+
+data_incorrect = """LUKS header information
+Version:       	2
+Epoch:         	8
+Metadata area: 	16384 [bytes]
+Keyslots area: 	16744448 [bytes]
+UUID:          	3e2649b4-88c9-4b3f-acf5-edcc380ecc23
+Label:         	(no label)
+Subsystem:     	(no subsystem)
+Flags:       	(no flags)
+
+Data segments:
+  0: crypt
+	offset: 16777216 [bytes]
+	length: (whole device)
+	cipher: blowfish
+	sector: 4096 [bytes]
+
+Keyslots:
+  0: luks2
+	Key:        256 bits
+	Priority:   normal
+	Cipher:     chacha-20
+	Cipher key: 256 bits
+	PBKDF:      argon2id
+	Time cost:  4
+	Memory:     1023865
+	Threads:    4
+	Salt:       ec e0 7a bf 23 d1 b0 13 ac 97 b3 8d 7b 6e 5c 4f
+	            21 a0 c1 19 f1 1e f7 0d 2b 67 3d e8 8a 4c 51 47
+	AF stripes: 4000
+	AF hash:    sha256
+	Area offset:32768 [bytes]
+	Area length:258048 [bytes]
+	Digest ID:  0
+  3: luks2
+	Key:        512 bits
+	Priority:   normal
+	Cipher:     aes-xts-plain64
+	Cipher key: 512 bits
+	PBKDF:      SHAKE382
+	Time cost:  4
+	Memory:     989547
+	Threads:    4
+	Salt:       7a 07 6e a9 20 ac 1e 8a 6b 87 4d 26 51 66 d1 ce
+	            f8 35 21 9a 2e 8d dc b6 9b d5 50 b8 f2 e1 11 3f
+	AF stripes: 4000
+	AF hash:    sha256
+	Area offset:290816 [bytes]
+	Area length:258048 [bytes]
+	Digest ID:  0
+Tokens:
+Digests:
+  0: pbkdf2
+	Hash:       sha256
+	Iterations: 75851
+	Salt:       3b 44 80 a2 6f 29 26 01 98 bb a2 92 cc 6e bc 7c
+	            df a9 e2 b2 90 ad 5e 4a e2 75 bf 9e ac 3f 81 59
+	Digest:     76 11 6a c0 53 28 aa 6c 89 a9 24 52 7a d9 51 39
+	            b4 d9 0e 91 50 2c 5a d4 ab df a2 6a 98 8f b1 ed"""
+
+
+def test_check_cipher_ok():
+    from fc.ceph.luks.checks import check_cipher
+
+    assert list(check_cipher(data_correct.splitlines())) == []
+
+
+def test_check_cipher_error():
+    from fc.ceph.luks.checks import check_cipher
+
+    assert list(check_cipher(data_incorrect.splitlines())) == [
+        "cipher: chacha-20 does not match aes-xts-plain64"
+    ]
+    assert list(check_cipher([])) == [
+        "Unable to check cipher correctness, no `Cipher:` found in dump"
+    ]
+    assert list(check_cipher(["gar", "bage"])) == [
+        "Unable to check cipher correctness, no `Cipher:` found in dump"
+    ]
+
+
+def test_check_key_slots_exactly_1_and_0_ok():
+    from fc.ceph.luks.checks import check_key_slots_exactly_1_and_0
+
+    assert (
+        list(check_key_slots_exactly_1_and_0(data_correct.splitlines())) == []
+    )
+
+
+def test_check_key_slots_exactly_1_and_0_error():
+    from fc.ceph.luks.checks import check_key_slots_exactly_1_and_0
+
+    assert list(
+        check_key_slots_exactly_1_and_0(data_incorrect.splitlines())
+    ) == ["keyslots: unexpected configuration ({0, 3})"]
+    assert list(check_key_slots_exactly_1_and_0([])) == [
+        "keyslots: unexpected configuration (set())"
+    ]
+    assert list(check_key_slots_exactly_1_and_0(["gar", "bage"])) == [
+        "keyslots: unexpected configuration (set())"
+    ]
+
+
+def test_check_512_bit_keys_ok():
+    from fc.ceph.luks.checks import check_512_bit_keys
+
+    assert list(check_512_bit_keys(data_correct.splitlines())) == []
+
+
+def test_check_512_bit_keys_error():
+    from fc.ceph.luks.checks import check_512_bit_keys
+
+    assert list(check_512_bit_keys(data_incorrect.splitlines())) == [
+        "keysize: 256 bits does not match expected 512 bits"
+    ]
+    assert list(check_512_bit_keys([])) == [
+        "Unable to check key size correctness, no `Key:` found in dump"
+    ]
+    assert list(check_512_bit_keys(["gar", "bage"])) == [
+        "Unable to check key size correctness, no `Key:` found in dump"
+    ]
+
+
+def test_check_pbkdf_is_argon2id_ok():
+    from fc.ceph.luks.checks import check_pbkdf_is_argon2id
+
+    assert list(check_pbkdf_is_argon2id(data_correct.splitlines())) == []
+
+
+def test_check_pbkdf_is_argon2id_error():
+    from fc.ceph.luks.checks import check_pbkdf_is_argon2id
+
+    assert list(check_pbkdf_is_argon2id(data_incorrect.splitlines())) == [
+        "pbkdf: SHAKE382 does not match expected argon2id"
+    ]
+    assert list(check_pbkdf_is_argon2id([])) == [
+        "Unable to check PBKDF correctness, no `PBKDF:` found in dump"
+    ]
+    assert list(check_pbkdf_is_argon2id(["gar", "bage"])) == [
+        "Unable to check PBKDF correctness, no `PBKDF:` found in dump"
+    ]
+
+
+def test_check_integration_ok(monkeypatch, capsys):
+    from fc.ceph.luks.manage import LuksDevice, LUKSKeyStoreManager
+
+    luksdevice_mock = MagicMock(
+        return_value=[
+            LuksDevice(
+                base_blockdev="/dev/mapper/foo",
+                name="testdev1",
+                mountpoint="/mnt/foo",
+            ),
+            LuksDevice(
+                base_blockdev="/dev/vgbar/holygrail",
+                name="testdev2",
+                mountpoint="/mnt/bar",
+            ),
+        ]
+    )
+    monkeypatch.setattr(LuksDevice, "filter_cryptvolumes", luksdevice_mock)
+
+    monkeypatch.setattr(
+        "fc.ceph.luks.Cryptsetup.cryptsetup",
+        MagicMock(return_value=data_correct.encode("utf-8")),
+    )
+
+    assert LUKSKeyStoreManager.check_luks("*", header=None) == 0
+
+    captured = capsys.readouterr()
+    assert captured.out == textwrap.dedent(
+        """\
+        Checking testdev1:
+        check_cipher: OK
+        check_key_slots_exactly_1_and_0: OK
+        check_512_bit_keys: OK
+        check_pbkdf_is_argon2id: OK
+        Checking testdev2:
+        check_cipher: OK
+        check_key_slots_exactly_1_and_0: OK
+        check_512_bit_keys: OK
+        check_pbkdf_is_argon2id: OK
+        """
+    )
+    assert captured.err == ""
+
+
+def test_check_integration_error(monkeypatch, capsys):
+    from fc.ceph.luks.manage import LuksDevice, LUKSKeyStoreManager
+
+    luksdevice_mock = MagicMock(
+        return_value=[
+            LuksDevice(
+                base_blockdev="/dev/mapper/foo",
+                name="testdev1",
+                mountpoint="/mnt/foo",
+            ),
+            LuksDevice(
+                base_blockdev="/dev/vgbar/holygrail",
+                name="testdev2",
+                mountpoint="/mnt/bar",
+            ),
+        ]
+    )
+    monkeypatch.setattr(LuksDevice, "filter_cryptvolumes", luksdevice_mock)
+
+    class DumpMock:
+        def __init__(self):
+            self.side_effects = iter([data_incorrect.encode("utf-8"), b""])
+
+        def __call__(self, *args, **kwargs):
+            return next(self.side_effects)
+
+    monkeypatch.setattr("fc.ceph.luks.Cryptsetup.cryptsetup", DumpMock())
+
+    assert LUKSKeyStoreManager.check_luks("*", header=None) == 1
+
+    captured = capsys.readouterr()
+    assert captured.out == textwrap.dedent(
+        """\
+        Checking testdev1:
+        check_cipher: cipher: chacha-20 does not match aes-xts-plain64
+        check_key_slots_exactly_1_and_0: keyslots: unexpected configuration ({0, 3})
+        check_512_bit_keys: keysize: 256 bits does not match expected 512 bits
+        check_pbkdf_is_argon2id: pbkdf: SHAKE382 does not match expected argon2id
+        Checking testdev2:
+        check_cipher: Unable to check cipher correctness, no `Cipher:` found in dump
+        check_key_slots_exactly_1_and_0: keyslots: unexpected configuration (set())
+        check_512_bit_keys: Unable to check key size correctness, no `Key:` found in """
+        + """
+        dump
+        check_pbkdf_is_argon2id: Unable to check PBKDF correctness, no `PBKDF:` found in
+        dump
+        """
+    )
+    assert captured.err == ""

From 0b78072757d350887fc2faa57ea61fc2e0f9e6d2 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Wed, 21 Aug 2024 21:58:45 +0200
Subject: [PATCH 07/13] fc-luks check: integrate smoke test into NixOS tests

This makes sense to expose the check to potential changes in to output
format of `cryptsetup luksDump`.
Detailed unit tests are done against mock outputs only.
---
 tests/backy_volumes.nix | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/backy_volumes.nix b/tests/backy_volumes.nix
index 2dbbefec9..ef8b0ce4e 100644
--- a/tests/backy_volumes.nix
+++ b/tests/backy_volumes.nix
@@ -45,7 +45,10 @@ in
     };
     newBacky = mkMachine {};
   };
-  testScript = {nodes, ...}:''
+  testScript = {nodes, ...}:
+    let
+    check_luksParams = testlib.sensuCheckCmd nodes.newBacky "luksParams";
+    in ''
     from time import sleep
 
 
@@ -110,5 +113,8 @@ in
       newBacky.succeed("${pkgs.util-linux}/bin/findmnt /srv/backy > /dev/kmsg 2>&1")
 
     test_reboot_automount(newBacky)
+
+    with subtest("Smoke test for LUKS metadata check"):
+      newBacky.succeed("${check_luksParams} > /dev/kmsg 2>&1")
   '';
 })

From 52bb07f1353199fca92e38bb730c69ebc0386c34 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Wed, 21 Aug 2024 23:15:11 +0200
Subject: [PATCH 08/13] fc-luks check: only enable sensu check on hosts with
 keystick

Not all physical machines actually have encrypted volumes as of now,
e.g. KVM hosts. The check does not need to run on them.
---
 nixos/platform/full-disk-encryption.nix | 2 +-
 pkgs/fc/ceph/src/fc/ceph/luks/manage.py | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 98f6fcb31..1588031d8 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -77,7 +77,7 @@ in
         luksParams = {
           notification = "LUKS Volumes use expected parameters.";
           interval = 3600;
-          command = "sudo ${check_luks_cmd}";
+          command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd}";
         };
       };
 
diff --git a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
index e735ed89b..177cdf258 100644
--- a/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
+++ b/pkgs/fc/ceph/src/fc/ceph/luks/manage.py
@@ -202,6 +202,10 @@ def check_luks(name_glob: str, header: Optional[str]) -> int:
                 f"Warning: The glob `{name_glob}` matches no volume.",
                 style="yellow",
             )
+            # based on the assumption that encrypted devices have to exist on a
+            # host in the normal case. Conditionalising the check to only run
+            # on hosts that are indeed expected to hav such devices needs to
+            # happen in platform code.
             return 1
 
         errors = 0

From 6de0aeecb718e38be2847e7e3f5fb7627e06ac58 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Wed, 21 Aug 2024 23:45:44 +0200
Subject: [PATCH 09/13] fc-luks check: fix sudoers rule for sensu check

---
 nixos/platform/full-disk-encryption.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 1588031d8..75bbae04a 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -30,7 +30,7 @@ let
       exit 0
     '';
   cephPkgs = fclib.ceph.mkPkgs "nautilus";  # FIXME: just a workaround
-  check_luks_cmd = "${cephPkgs.fc-ceph}/bin/fc-luks check '*'";
+  check_luks_cmd = "${cephPkgs.fc-ceph}/bin/fc-luks check";
 in
 {
 
@@ -77,12 +77,12 @@ in
         luksParams = {
           notification = "LUKS Volumes use expected parameters.";
           interval = 3600;
-          command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd}";
+          command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd} '*'";
         };
       };
 
       flyingcircus.passwordlessSudoRules = [{
-        commands = [(toString check_key_file) check_luks_cmd];
+        commands = [(toString check_key_file) "${check_luks_cmd} *"];
         groups = ["sensuclient"];
       }];
 

From e47b34bef3f1cdd07d192b74ec205061838cf3a2 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Thu, 22 Aug 2024 12:58:23 +0200
Subject: [PATCH 10/13] fc-luks: read PATH from config file to fix sensu check
 run

By running fc-luks commands as a sensu check, we suddenly do not inherit
the full system PATH anymore, breaking access to necessary external
tools used like `lvs`.
We can adopt the `fc-ceph.conf` approach already used by `fc-ceph`
subsystems, for now even the default PATH is sufficient.

Requires some extra mocking in unit tests; the NixOS integration test
now explicitly empties the path beforehands.
---
 nixos/lib/ceph-common.nix                   |  1 +
 pkgs/fc/ceph/src/fc/ceph/main.py            |  3 ++-
 pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py | 18 +++++++++++++++++-
 tests/backy_volumes.nix                     |  2 +-
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/nixos/lib/ceph-common.nix b/nixos/lib/ceph-common.nix
index 72b4bd133..cb33d1bd3 100644
--- a/nixos/lib/ceph-common.nix
+++ b/nixos/lib/ceph-common.nix
@@ -36,6 +36,7 @@ rec {
       pkgs.coreutils
       pkgs.lz4  # required by image loading task
       pkgs.cryptsetup  # full-disk encryption
+      pkgs.mdadm  # fc-luks, backup RAID
     ];
 
     fc-check-ceph = pkgs.fc."check-ceph-${release}";
diff --git a/pkgs/fc/ceph/src/fc/ceph/main.py b/pkgs/fc/ceph/src/fc/ceph/main.py
index 069e7f0ea..addcdf68d 100644
--- a/pkgs/fc/ceph/src/fc/ceph/main.py
+++ b/pkgs/fc/ceph/src/fc/ceph/main.py
@@ -571,7 +571,8 @@ def luks(args=sys.argv[1:]):
         action()
         sys.exit(1)
 
-    subsystem = subsystem_factory()
+    environment = Environment(CONFIG_FILE_PATH)
+    subsystem = environment.prepare(subsystem_factory)
     action = getattr(subsystem, action)
     action_statuscode = action(**args)
 
diff --git a/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
index 8a86f2889..9ee2c8e74 100644
--- a/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
+++ b/pkgs/fc/ceph/src/fc/ceph/tests/test_luks.py
@@ -561,11 +561,27 @@ def test_keystore_admin_key_fingerprint_existing_update(
 
 
 def test_luks_fingerprint_command_invocation(
-    inputs_mock, capsys, mock_LUKSKeyStoreManager
+    inputs_mock, capsys, mock_LUKSKeyStoreManager, monkeypatch, tmp_path
 ):
     """smoke test for invocation via CLI arguments"""
     from fc.ceph.main import luks
 
+    monkeypatch.setattr(
+        "fc.ceph.main.CONFIG_FILE_PATH", tmp_path / "fc-ceph.conf"
+    )
+
+    with open(tmp_path / "fc-ceph.conf", "wt") as configfilemock:
+        print(
+            dedent(
+                """\
+        [default]
+        path=/nix/store/qkb2vanjhgvhyn9c6d7nab97dp956cqn-ceph-14.2.22/bin:/nix/store/czcdjj2yjm3bdaamv2212ph11i446ldj-ceph-client-14.2.22/bin:/nix/store/qnwhmvhhs0kdd849zisbs0na0qr52qg4-xfsprogs-5.11.0-bin/bin:/nix/store/ljrwk0h6qifb661lc5mjz6vd712r05cd-lvm2-2.03.12-bin/bin:/nix/store/bfxkg8q43b5hbzhzmq6c5j2iqf6806a5-util-linux-2.36.2-bin/bin:/nix/store/6pzvibxjz7pdi1dgn1f2dav4b2fhdyga-systemd-247.6/bin:/nix/store/xa76ngikl4sqbk18f0zwhhxa9lfay3zq-gptfdisk-1.0.7/bin:/nix/store/y41s1vcn0irn9ahn9wh62yx2cygs7qjj-coreutils-8.32/bin:/nix/store/2w2v7y95ll8896kaybxhnaiv23rb7q9n-lz4-1.9.3-bin/bin:/nix/store/3j44b29gyrdkpvkj5cfn2cywgcx440ms-cryptsetup-2.3.5/bin
+        release=nautilus
+        """
+            ),
+            file=configfilemock,
+        )
+
     # feed data to `input()`
     inputs_mock.write(
         "\n".join(
diff --git a/tests/backy_volumes.nix b/tests/backy_volumes.nix
index ef8b0ce4e..ab236cb53 100644
--- a/tests/backy_volumes.nix
+++ b/tests/backy_volumes.nix
@@ -115,6 +115,6 @@ in
     test_reboot_automount(newBacky)
 
     with subtest("Smoke test for LUKS metadata check"):
-      newBacky.succeed("${check_luksParams} > /dev/kmsg 2>&1")
+      newBacky.succeed("(export PATH='${pkgs.sudo}/bin/'; ${check_luksParams} > /dev/kmsg 2>&1 )")
   '';
 })

From 241ad1285857aa2af7b494990ff3412604046790 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Thu, 22 Aug 2024 13:37:11 +0200
Subject: [PATCH 11/13] make nixd happy

---
 nixos/services/ceph/client.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/services/ceph/client.nix b/nixos/services/ceph/client.nix
index 58069bd56..5198310f2 100644
--- a/nixos/services/ceph/client.nix
+++ b/nixos/services/ceph/client.nix
@@ -45,7 +45,7 @@ let
     monCompactOnStart = true;     # Keep mondb small
     monHost = mons;
     monOsdDownOutInterval = 900;  # Allow 15 min for reboots to happen without backfilling.
-    monOsdNearfullRatio = .9;
+    monOsdNearfullRatio = 0.9;
 
     monData = "/srv/ceph/mon/$cluster-$id";
     monOsdAllowPrimaryAffinity = true;

From eee1aceb62d15f378a2949d2275ab6d0498b4d0c Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Thu, 22 Aug 2024 13:38:08 +0200
Subject: [PATCH 12/13] fc-ceph.conf: refactor to enable it on non-ceph client
 hosts for fc-luks

---
 nixos/platform/full-disk-encryption.nix | 16 +++++++-----
 nixos/services/ceph/client.nix          | 34 +++++++++++++++++--------
 nixos/services/ceph/server.nix          |  5 +++-
 3 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 75bbae04a..8b9d539aa 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -29,8 +29,7 @@ let
 
       exit 0
     '';
-  cephPkgs = fclib.ceph.mkPkgs "nautilus";  # FIXME: just a workaround
-  check_luks_cmd = "${cephPkgs.fc-ceph}/bin/fc-luks check";
+  check_luks_cmd = "${config.flyingcircus.services.ceph.fc-ceph.package}/bin/fc-luks check";
 in
 {
 
@@ -49,14 +48,17 @@ in
   };
 
   config = lib.mkIf (config.flyingcircus.infrastructureModule == "flyingcircus-physical" ||
+    # TODO: When merging nixos-hardware with our regular VM branch, we need to refine this
+    # to avoid that all regular VM tests (e.g. PHP) get fc-luks cruft added.
     config.flyingcircus.infrastructureModule == "testing"
     )
   {
-      environment.systemPackages = with pkgs; [
-        cryptsetup
-        # FIXME: isolate fc-luks tooling into separate package
-        cephPkgs.fc-ceph
-      ];
+    environment.systemPackages = with pkgs; [
+      cryptsetup
+    ];
+
+    # FIXME: isolate fc-luks tooling into separate package
+    flyingcircus.services.ceph.fc-ceph.enable = true;
 
       flyingcircus.services.sensu-client.checks = {
         keystickMounted = {
diff --git a/nixos/services/ceph/client.nix b/nixos/services/ceph/client.nix
index 5198310f2..29c2baee0 100644
--- a/nixos/services/ceph/client.nix
+++ b/nixos/services/ceph/client.nix
@@ -114,11 +114,15 @@ in
       };
 
       fc-ceph = {
+        enable = lib.mkEnableOption "enable fc-ceph command and supporting infrastracture (also contains fc-luks)";
         settings = lib.mkOption {
           type = with lib.types; attrsOf (attrsOf (oneOf [ bool int str package ]));
-          default = { };
           description = "Configuration for the fc-ceph utility, will be turned into the contents of /etc/ceph/fc-ceph.conf";
         };
+        package = lib.mkOption {
+          type = lib.types.package;
+          default = cephPkgs.fc-ceph;
+        };
       };
 
       client = {
@@ -152,7 +156,8 @@ in
     };
   };
 
-  config = lib.mkIf cfg.client.enable {
+  config = lib.mkMerge [
+    (lib.mkIf cfg.client.enable {
 
     assertions = [
       {
@@ -161,14 +166,7 @@ in
       }
     ];
 
-    # config file to be read by fc-ceph
-    environment.etc."ceph/fc-ceph.conf".text = lib.generators.toINI { } cfg.fc-ceph.settings;
 
-    # build a default binary path for fc-ceph
-    flyingcircus.services.ceph.fc-ceph.settings.default = {
-      release = cfg.client.cephRelease;
-      path = cephPkgs.fc-ceph-path;
-    };
     environment.systemPackages = [ cfg.client.package ];
 
     boot.kernelModules = [ "rbd" ];
@@ -220,6 +218,22 @@ in
       }
     '';
 
-  };
+  })
+  # fc-ceph can be enabled separately from the whole ceph (client)
+  # infrastructure, as it contains `fc-luks` for now which might be required on
+  # non-ceph hosts.
+  (lib.mkIf cfg.fc-ceph.enable {
+
+    # config file to be read by fc-ceph
+    environment.etc."ceph/fc-ceph.conf".text = lib.generators.toINI { } cfg.fc-ceph.settings;
+
+    # build a default binary path for fc-ceph
+    flyingcircus.services.ceph.fc-ceph.settings.default = {
+      release = cfg.client.cephRelease;
+      path = cephPkgs.fc-ceph-path;
+    };
+    environment.systemPackages = [ cfg.fc-ceph.package ];
+
+  })];
 
 }
diff --git a/nixos/services/ceph/server.nix b/nixos/services/ceph/server.nix
index c20435b47..c2e2dc3dd 100644
--- a/nixos/services/ceph/server.nix
+++ b/nixos/services/ceph/server.nix
@@ -107,8 +107,11 @@ in
       }
     ];
 
+    flyingcircus.services.ceph.fc-ceph = {
+      enable = true;
+      package = cephPkgs.fc-ceph;
+    };
     environment.systemPackages = with pkgs; [
-      cephPkgs.fc-ceph
       fc.blockdev
 
       # tools like radosgw-admin and crushtool are only included in the full ceph package, but are necessary admin tools

From f5c854ee1a29a6f0a188e94d1df336394df15356 Mon Sep 17 00:00:00 2001
From: Oliver Schmidt <os@flyingcircus.io>
Date: Thu, 22 Aug 2024 13:53:14 +0200
Subject: [PATCH 13/13] [formatting] remove unnecessary indentation

---
 nixos/platform/full-disk-encryption.nix | 52 ++++++++++++-------------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/nixos/platform/full-disk-encryption.nix b/nixos/platform/full-disk-encryption.nix
index 8b9d539aa..764dd8f7e 100644
--- a/nixos/platform/full-disk-encryption.nix
+++ b/nixos/platform/full-disk-encryption.nix
@@ -60,35 +60,35 @@ in
     # FIXME: isolate fc-luks tooling into separate package
     flyingcircus.services.ceph.fc-ceph.enable = true;
 
-      flyingcircus.services.sensu-client.checks = {
-        keystickMounted = {
-          notification = "USB stick with disk encryption keys is mounted and keyfile is readable.";
-          interval = 60;
-          command = "sudo ${check_key_file}";
-        };
-        noSwap = {
-          notification = "Machine does not use swap to arbitrarily persist memory pages with sensitive data.";
-          interval = 60;
-          command = toString (pkgs.writeShellScript "noSwapCheck" ''
-            # /proc/swaps always has a header line
-            if [ $(${pkgs.coreutils}/bin/cat /proc/swaps | ${pkgs.coreutils}/bin/wc -l) -ne 1 ]; then
-              exit 1
-            fi
-          '');
-        };
-        luksParams = {
-          notification = "LUKS Volumes use expected parameters.";
-          interval = 3600;
-          command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd} '*'";
-        };
+    flyingcircus.services.sensu-client.checks = {
+      keystickMounted = {
+        notification = "USB stick with disk encryption keys is mounted and keyfile is readable.";
+        interval = 60;
+        command = "sudo ${check_key_file}";
       };
+      noSwap = {
+        notification = "Machine does not use swap to arbitrarily persist memory pages with sensitive data.";
+        interval = 60;
+        command = toString (pkgs.writeShellScript "noSwapCheck" ''
+          # /proc/swaps always has a header line
+          if [ $(${pkgs.coreutils}/bin/cat /proc/swaps | ${pkgs.coreutils}/bin/wc -l) -ne 1 ]; then
+            exit 1
+          fi
+        '');
+      };
+      luksParams = {
+        notification = "LUKS Volumes use expected parameters.";
+        interval = 3600;
+        command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd} '*'";
+      };
+    };
 
-      flyingcircus.passwordlessSudoRules = [{
-        commands = [(toString check_key_file) "${check_luks_cmd} *"];
-        groups = ["sensuclient"];
-      }];
+    flyingcircus.passwordlessSudoRules = [{
+      commands = [(toString check_key_file) "${check_luks_cmd} *"];
+      groups = ["sensuclient"];
+    }];
 
-      fileSystems.${keysMountDir} = config.flyingcircus.infrastructure.fullDiskEncryption.fsOptions;
+    fileSystems.${keysMountDir} = config.flyingcircus.infrastructure.fullDiskEncryption.fsOptions;
   };
 
 }