[21.05] full-disk encryption: finalisation, improvements, checks, routine tooling

nixos/infrastructure/flyingcircus-physical.nix

@@ -4,7 +4,6 @@ with lib;
 
 let
   cfg = config.flyingcircus;
-  inherit (config) fclib;
 in
 mkIf (cfg.infrastructureModule == "flyingcircus-physical") {
 

nixos/lib/ceph-common.nix

@@ -36,6 +36,7 @@ rec {
       pkgs.coreutils
       pkgs.lz4  # required by image loading task
       pkgs.cryptsetup  # full-disk encryption
+      pkgs.mdadm  # fc-luks, backup RAID
     ];
 
     fc-check-ceph = pkgs.fc."check-ceph-${release}";

nixos/platform/full-disk-encryption.nix

@@ -29,8 +29,7 @@ let
 
       exit 0
     '';
-
-  cephPkgs = fclib.ceph.mkPkgs "nautilus";  # FIXME: just a workaround
+  check_luks_cmd = "${config.flyingcircus.services.ceph.fc-ceph.package}/bin/fc-luks check";
 in
 {
 
@@ -49,27 +48,47 @@ in
   };
 
   config = lib.mkIf (config.flyingcircus.infrastructureModule == "flyingcircus-physical" ||
+    # TODO: When merging nixos-hardware with our regular VM branch, we need to refine this
+    # to avoid that all regular VM tests (e.g. PHP) get fc-luks cruft added.
     config.flyingcircus.infrastructureModule == "testing"
     )
   {
-      environment.systemPackages = with pkgs; [
-        cryptsetup
-        # FIXME: isolate fc-luks tooling into separate package
-        cephPkgs.fc-ceph
-      ];
+    environment.systemPackages = with pkgs; [
+      cryptsetup
+    ];
+
+    # FIXME: isolate fc-luks tooling into separate package
+    flyingcircus.services.ceph.fc-ceph.enable = true;
 
-      flyingcircus.services.sensu-client.checks.keystickMounted = {
+    flyingcircus.services.sensu-client.checks = {
+      keystickMounted = {
         notification = "USB stick with disk encryption keys is mounted and keyfile is readable.";
         interval = 60;
         command = "sudo ${check_key_file}";
       };
+      noSwap = {
+        notification = "Machine does not use swap to arbitrarily persist memory pages with sensitive data.";
+        interval = 60;
+        command = toString (pkgs.writeShellScript "noSwapCheck" ''
+          # /proc/swaps always has a header line
+          if [ $(${pkgs.coreutils}/bin/cat /proc/swaps | ${pkgs.coreutils}/bin/wc -l) -ne 1 ]; then
+            exit 1
+          fi
+        '');
+      };
+      luksParams = {
+        notification = "LUKS Volumes use expected parameters.";
+        interval = 3600;
+        command = "test ! -d ${keysMountDir} || sudo ${check_luks_cmd} '*'";
+      };
+    };
 
-      flyingcircus.passwordlessSudoRules = [{
-        commands = [(toString check_key_file)];
-        groups = ["sensuclient"];
-      }];
+    flyingcircus.passwordlessSudoRules = [{
+      commands = [(toString check_key_file) "${check_luks_cmd} *"];
+      groups = ["sensuclient"];
+    }];
 
-      fileSystems.${keysMountDir} = config.flyingcircus.infrastructure.fullDiskEncryption.fsOptions;
+    fileSystems.${keysMountDir} = config.flyingcircus.infrastructure.fullDiskEncryption.fsOptions;
   };
 
 }

nixos/services/ceph/client.nix

@@ -45,7 +45,7 @@ let
     monCompactOnStart = true;     # Keep mondb small
     monHost = mons;
     monOsdDownOutInterval = 900;  # Allow 15 min for reboots to happen without backfilling.
-    monOsdNearfullRatio = .9;
+    monOsdNearfullRatio = 0.9;
 
     monData = "/srv/ceph/mon/$cluster-$id";
     monOsdAllowPrimaryAffinity = true;
@@ -114,11 +114,15 @@ in
       };
 
       fc-ceph = {
+        enable = lib.mkEnableOption "enable fc-ceph command and supporting infrastracture (also contains fc-luks)";
         settings = lib.mkOption {
           type = with lib.types; attrsOf (attrsOf (oneOf [ bool int str package ]));
-          default = { };
           description = "Configuration for the fc-ceph utility, will be turned into the contents of /etc/ceph/fc-ceph.conf";
         };
+        package = lib.mkOption {
+          type = lib.types.package;
+          default = cephPkgs.fc-ceph;
+        };
       };
 
       client = {
@@ -152,7 +156,8 @@ in
     };
   };
 
-  config = lib.mkIf cfg.client.enable {
+  config = lib.mkMerge [
+    (lib.mkIf cfg.client.enable {
 
     assertions = [
       {
@@ -161,14 +166,7 @@ in
       }
     ];
 
-    # config file to be read by fc-ceph
-    environment.etc."ceph/fc-ceph.conf".text = lib.generators.toINI { } cfg.fc-ceph.settings;
 
-    # build a default binary path for fc-ceph
-    flyingcircus.services.ceph.fc-ceph.settings.default = {
-      release = cfg.client.cephRelease;
-      path = cephPkgs.fc-ceph-path;
-    };
     environment.systemPackages = [ cfg.client.package ];
 
     boot.kernelModules = [ "rbd" ];
@@ -220,6 +218,22 @@ in
       }
     '';
 
-  };
+  })
+  # fc-ceph can be enabled separately from the whole ceph (client)
+  # infrastructure, as it contains `fc-luks` for now which might be required on
+  # non-ceph hosts.
+  (lib.mkIf cfg.fc-ceph.enable {
+
+    # config file to be read by fc-ceph
+    environment.etc."ceph/fc-ceph.conf".text = lib.generators.toINI { } cfg.fc-ceph.settings;
+
+    # build a default binary path for fc-ceph
+    flyingcircus.services.ceph.fc-ceph.settings.default = {
+      release = cfg.client.cephRelease;
+      path = cephPkgs.fc-ceph-path;
+    };
+    environment.systemPackages = [ cfg.fc-ceph.package ];
+
+  })];
 
 }

nixos/services/ceph/server.nix

@@ -107,8 +107,11 @@ in
       }
     ];
 
+    flyingcircus.services.ceph.fc-ceph = {
+      enable = true;
+      package = cephPkgs.fc-ceph;
+    };
     environment.systemPackages = with pkgs; [
-      cephPkgs.fc-ceph
       fc.blockdev
 
       # tools like radosgw-admin and crushtool are only included in the full ceph package, but are necessary admin tools

pkgs/fc/ceph/src/fc/ceph/luks/checks.py

@@ -0,0 +1,89 @@
+from typing import Iterator
+
+# In case these checks ever break: More recent cryptsetup versions also support
+# JSON-formatted output via `cryptsetup luksDump --dump-json-metadata <dev>`.
+# It most likely makes sense parsing that data instead once available.
+
+
+def check_cipher(lines: str) -> Iterator[str]:
+    checked = False  # plausibility check: did we get any cipher info?
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Cipher:"):
+            continue
+        cipher = line.split(":")[1].strip()
+        if cipher != "aes-xts-plain64":
+            yield f"cipher: {cipher} does not match aes-xts-plain64"
+        checked = True
+
+    if not checked:
+        yield "Unable to check cipher correctness, no `Cipher:` found in dump"
+
+
+def _extract_keyslot_numbers(lines: str):
+    known_keyslots: set[int] = set()
+    lines_iter = iter(lines)
+    for line in lines_iter:
+        if line.startswith("Keyslots:"):
+            break
+    else:
+        return known_keyslots
+
+    for line in lines_iter:
+        if line.startswith("Tokens:"):
+            break
+        if not ":" in line:
+            continue
+        header, value = line.split(":")
+        try:
+            header_i = int(header.strip())
+        except Exception:
+            continue
+        assert value.strip() == "luks2", (header, value, line)
+        known_keyslots.add(header_i)
+    return known_keyslots
+
+
+def check_key_slots_exactly_1_and_0(lines: str) -> Iterator[str]:
+    keyslots = _extract_keyslot_numbers(lines)
+    if set([0, 1]) != keyslots:
+        yield f"keyslots: unexpected configuration ({keyslots})"
+
+
+def check_512_bit_keys(lines: str) -> Iterator[str]:
+    checked = False
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("Key:"):
+            continue
+        key_size = line.split(":")[1].strip()
+        if key_size != "512 bits":
+            yield f"keysize: {key_size} does not match expected 512 bits"
+        checked = True
+
+    if not checked:
+        yield "Unable to check key size correctness, no `Key:` found in dump"
+
+
+def check_pbkdf_is_argon2id(lines: str) -> Iterator[str]:
+    checked = False
+    for line in lines:
+        line = line.strip()
+        if not line.startswith("PBKDF:"):
+            continue
+        pbkdf = line.split(":")[1].strip()
+        if pbkdf != "argon2id":
+            yield f"pbkdf: {pbkdf} does not match expected argon2id"
+        checked = True
+
+    if not checked:
+        yield "Unable to check PBKDF correctness, no `PBKDF:` found in dump"
+
+
+# All these checks work on a list of lines output by `cryptsetup luksDump`
+all_checks = [
+    check_cipher,
+    check_key_slots_exactly_1_and_0,
+    check_512_bit_keys,
+    check_pbkdf_is_argon2id,
+]