snapshot

flyingcircusio · Oct 5, 2023 · c80b4c4 · c80b4c4
1 parent 238ba23
commit c80b4c4
Show file tree

Hide file tree

Showing 24 changed files with 267 additions and 543 deletions.
diff --git a/nixos/infrastructure/flyingcircus-physical.nix b/nixos/infrastructure/flyingcircus-physical.nix
@@ -45,12 +45,11 @@ mkIf (cfg.infrastructureModule == "flyingcircus-physical") {
     environment.systemPackages = with pkgs; [
       fc.ledtool
       fc.secure-erase
+      fc.util-physical
       mstflint
+      nvme-cli
       pciutils
       smartmontools
-      nvme-cli
-      # ensure that `rbd-locktool` uses the correct ceph tooling version
-      config.fclib.ceph.releasePkgs.${cfg.services.ceph.client.cephRelease}.utilPhysical
     ];
 
     fileSystems = {

diff --git a/nixos/lib/ceph-common.nix b/nixos/lib/ceph-common.nix
@@ -4,42 +4,45 @@
 with lib;
 let
   # supported ceph release codenames, from newest to oldest
-  # TODO: Once all ceph packages have a similar structure, releasePkgs can be
+  # TODO: Once all ceph packages have a similar structure, those can be
   # generated from this list (let's wait for pacific to see if the structure holds)
   releaseOrder = [ "nautilus" ];
-  cephReleaseType = types.enum (builtins.attrNames releasePkgs);
+  cephReleaseType = types.enum releaseOrder;
   defaultRelease = "nautilus";
-
-  # ====== mapping of packages per ceph release =======
-
-  releasePkgs = {
-    "nautilus" = rec {
-      ceph = pkgs.ceph-nautilus.ceph;
-      ceph-client = pkgs.ceph-nautilus.ceph-client;
-      # both the C lib and the python modules
-      libceph = pkgs.ceph-nautilus.libceph;
-      fcQemu = pkgs.fc.qemu.override {
-        inherit libceph ceph;
-        qemu_ceph = qemu_ceph_versioned "nautilus";
-      };
-      utilPhysical = pkgs.fc.util-physical.nautilus.override {ceph = ceph-client;};
-    };
-  };
-  qemu_ceph_versioned = cephReleaseName: (pkgs.qemu_ceph.override {
-     # Needs full ceph package, because
-     #`libceph` is missing the dev output headers
-     ceph = releasePkgs.${cephReleaseName}.ceph;
-     });
 in
 rec {
   # constants
-  inherit releasePkgs defaultRelease qemu_ceph_versioned;
+  inherit defaultRelease;
   releaseOption = lib.mkOption {
     type = cephReleaseType;
     # centrally manage the default release for all roles here
     default = defaultRelease;
   };
 
+  mkPkgs = release: rec {
+    ceph = pkgs."ceph-${release}".ceph;
+    ceph-client = pkgs."ceph-${release}".ceph-client;
+    libceph = pkgs."ceph-${release}".libceph;
+
+    fc-ceph = pkgs.fc.ceph;
+    fc-ceph-path = lib.makeBinPath [
+      ceph
+      ceph-client
+      pkgs.xfsprogs
+      pkgs.lvm2
+      pkgs.util-linux
+      pkgs.systemd
+      pkgs.gptfdisk
+      pkgs.coreutils
+      pkgs.lz4  # required by image loading task
+    ];
+
+    fc-check-ceph = pkgs.fc."check-ceph-${release}";
+
+    fc-qemu = pkgs.fc."qemu-${release}";
+    qemu = pkgs."qemu-ceph-${release}";
+  };
+
   # helper functions
 
   # returns the highest supported release encountered in any of the active roles
@@ -68,21 +71,6 @@ rec {
       else _releaseRecurser (builtins.tail rList) acc;
     in _releaseRecurser (lib.reverseList releaseOrder) false;
 
-
-  # return a suitable binary path for fc-ceph, parameterised with the desired ceph package
-  fc-ceph-path = cephPkg: lib.makeBinPath [
-    cephPkg
-    pkgs.xfsprogs
-    pkgs.lvm2
-    pkgs.util-linux
-    pkgs.systemd
-    pkgs.gptfdisk
-    pkgs.coreutils
-    releasePkgs.${cephPkg.codename}.utilPhysical # required for rbd-locktool
-    pkgs.lz4  # required by image loading task
-    ];
-
-
   # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
   expandCamelCase = lib.replaceStrings lib.upperChars (map (s: " ${s}") lib.lowerChars);
   expandCamelCaseAttrs = lib.mapAttrs' (name: value: lib.nameValuePair (expandCamelCase name) value);

diff --git a/nixos/roles/backyserver.nix b/nixos/roles/backyserver.nix
@@ -22,8 +22,10 @@ let
 
   restoreSingleFiles = pkgs.callPackage ../../pkgs/restore-single-files {};
 
-  backyRbdVersioned = cephReleaseName: {
-    BACKY_RBD = "${fclib.ceph.releasePkgs.${cephReleaseName}.ceph-client}/bin/rbd";
+  cephPkgs = fclib.ceph.mkPkgs role.cephRelease;
+
+  backyRbdVersioned = {
+    BACKY_RBD = "${cephPkgs.ceph-client}/bin/rbd";
   };
 
 in
@@ -64,7 +66,7 @@ in
     ];
 
     # globally set the RBD to be used by backy, in case it is invoked manually by an operator
-    environment.variables = backyRbdVersioned role.cephRelease;
+    environment.variables = backyRbdVersioned;
 
     fileSystems = {
       "/srv/backy" = {
@@ -121,7 +123,7 @@ in
 
         environment = {
           CEPH_ARGS = "--id ${enc.name}";
-        } // backyRbdVersioned role.cephRelease;
+        } // backyRbdVersioned;
 
         serviceConfig = {
           ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";

diff --git a/nixos/roles/ceph/mon.nix b/nixos/roles/ceph/mon.nix
@@ -12,10 +12,7 @@ let
   # We do not have service data during bootstrapping.
   first_mon = if mons == [ ] then "" else head (lib.splitString "." (head mons));
 
-  # TODO: once all ceph releases use the ceph-client attr name, ensure that the desired
-  # build is used here by explicitly overriding/ passing it here.
-  fc-check-ceph-withVersion = pkgs.fc.check-ceph.${role.cephRelease};
-  fc-ceph = pkgs.fc.cephWith fclib.ceph.releasePkgs.${role.cephRelease}.ceph;
+  cephPkgs = fclib.ceph.mkPkgs role.cephRelease;
 
   # default definitions for the mgr.* options:
   mgrEnabledModules = {
@@ -122,7 +119,7 @@ in
         fc-ceph.settings = let
           monSettings =  {
             release = role.cephRelease;
-            path = fclib.ceph.fc-ceph-path fclib.ceph.releasePkgs.${role.cephRelease}.ceph;
+            path = cephPkgs.fc-ceph-path;
           };
         in {
           # fc-ceph monitor components
@@ -149,23 +146,23 @@ in
 
         restartTriggers = [
           config.environment.etc."ceph/ceph.conf".source
-          fclib.ceph.releasePkgs.${role.cephRelease}.ceph
+          cephPkgs.ceph
         ];
 
         environment = {
           PYTHONUNBUFFERED = "1";
         };
 
         script = ''
-          ${fc-ceph}/bin/fc-ceph mon activate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mon activate
         '';
 
         reload = ''
-          ${fc-ceph}/bin/fc-ceph mon reactivate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mon reactivate
         '';
 
         preStop = ''
-          ${fc-ceph}/bin/fc-ceph mon deactivate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mon deactivate
         '';
 
         serviceConfig = {
@@ -177,19 +174,19 @@ in
       flyingcircus.passwordlessSudoRules = [
         {
           commands = [
-            "${fc-check-ceph-withVersion}/bin/check_ceph"
-            "${fc-check-ceph-withVersion}/bin/check_snapshot_restore_fill"
+            "${cephPkgs.fc-check-ceph}/bin/check_ceph"
+            "${cephPkgs.fc-check-ceph}/bin/check_snapshot_restore_fill"
           ];
           groups = [ "sensuclient" ];
         }
       ];
 
-      environment.systemPackages = [ fc-check-ceph-withVersion ];
+      environment.systemPackages = [ cephPkgs.fc-check-ceph ];
 
       systemd.services.fc-ceph-load-vm-images = {
         description = "Load new VM base images";
         serviceConfig.Type = "oneshot";
-        script = "${fc-ceph}/bin/fc-ceph maintenance load-vm-images";
+        script = "${cephPkgs.fc-ceph}/bin/fc-ceph maintenance load-vm-images";
         environment = {
           PYTHONUNBUFFERED = "1";
         };
@@ -198,7 +195,7 @@ in
       systemd.services.fc-ceph-purge-old-snapshots = {
         description = "Purge old snapshots";
         serviceConfig.Type = "oneshot";
-        script = "${fc-ceph}/bin/fc-ceph maintenance purge-old-snapshots";
+        script = "${cephPkgs.fc-ceph}/bin/fc-ceph maintenance purge-old-snapshots";
         environment = {
           PYTHONUNBUFFERED = "1";
         };
@@ -207,7 +204,7 @@ in
       systemd.services.fc-ceph-clean-deleted-vms = {
         description = "Purge old snapshots";
         serviceConfig.Type = "oneshot";
-        script = "${fc-ceph}/bin/fc-ceph maintenance clean-deleted-vms";
+        script = "${cephPkgs.fc-ceph}/bin/fc-ceph maintenance clean-deleted-vms";
         environment = {
           PYTHONUNBUFFERED = "1";
         };
@@ -216,7 +213,7 @@ in
       systemd.services.fc-ceph-mon-update-client-keys = {
         description = "Update client keys and authorization in the monitor database.";
         serviceConfig.Type = "oneshot";
-        script = "${fc-ceph}/bin/fc-ceph keys mon-update-client-keys";
+        script = "${cephPkgs.fc-ceph}/bin/fc-ceph keys mon-update-client-keys";
         environment = {
           PYTHONUNBUFFERED = "1";
         };
@@ -232,30 +229,30 @@ in
 
         restartTriggers = [
           config.environment.etc."ceph/ceph.conf".source
-          fclib.ceph.releasePkgs.${role.cephRelease}.ceph
+          cephPkgs.ceph
         ];
 
         environment = {
           PYTHONUNBUFFERED = "1";
         };
 
         script = ''
-          ${fc-ceph}/bin/fc-ceph mgr activate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mgr activate
         '';
 
         reload = ''
-          ${fc-ceph}/bin/fc-ceph mgr reactivate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mgr reactivate
         '';
 
         # imperatively ensure mgr modules
         preStart = lib.concatStringsSep "\n" (
-          lib.forEach mgrEnabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module enable ${mod} --force")
+          lib.forEach mgrEnabledModules.${role.cephRelease} (mod: "${cephPkgs.ceph}/bin/ceph mgr module enable ${mod} --force")
           ++
-          lib.forEach mgrDisabledModules.${role.cephRelease} (mod: "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/ceph mgr module disable ${mod}")
+          lib.forEach mgrDisabledModules.${role.cephRelease} (mod: "${cephPkgs.ceph}/bin/ceph mgr module disable ${mod}")
         );
 
         preStop = ''
-          ${fc-ceph}/bin/fc-ceph mgr deactivate
+          ${cephPkgs.fc-ceph}/bin/fc-ceph mgr deactivate
         '';
 
         serviceConfig = {
@@ -278,12 +275,12 @@ in
           ceph_snapshot_restore_fill = {
             notification = "The Ceph cluster might not have enough space for restoring "
               + "the largest RBD snapshot. (does not consider sparse allocation)";
-            command = "sudo ${fc-check-ceph-withVersion}/bin/check_snapshot_restore_fill ${configtoml}";
+            command = "sudo ${cephPkgs.fc-check-ceph}/bin/check_snapshot_restore_fill ${configtoml}";
             interval = 600;
           };
           ceph = {
             notification = "Ceph cluster is unhealthy";
-            command = "sudo ${fc-check-ceph-withVersion}/bin/check_ceph -v -R 200 -A 300";
+            command = "sudo ${cephPkgs.fc-check-ceph}/bin/check_ceph -v -R 200 -A 300";
             interval = 60;
           };
         };

diff --git a/nixos/roles/ceph/osd.nix b/nixos/roles/ceph/osd.nix
@@ -8,7 +8,7 @@ let
   enc = config.flyingcircus.enc;
   inherit (fclib.ceph) expandCamelCaseAttrs expandCamelCaseSection;
 
-  fc-ceph = pkgs.fc.cephWith fclib.ceph.releasePkgs.${role.cephRelease}.ceph;
+  cephPkgs = fclib.ceph.mkPkgs role.cephRelease;
 
   defaultOsdSettings = {
     # Assist speedy but balanced recovery
@@ -170,7 +170,7 @@ in
         fc-ceph.settings = let
           osdSettings =  {
             release = role.cephRelease;
-            path = fclib.ceph.fc-ceph-path fclib.ceph.releasePkgs.${role.cephRelease}.ceph;
+            path = cephPkgs.fc-ceph-path;
           };
         in {
           # fc-ceph OSD
@@ -199,15 +199,15 @@ in
         restartIfChanged = false;
 
         script = ''
-          ${fc-ceph}/bin/fc-ceph osd activate all
+          ${cephPkgs.fc-ceph}/bin/fc-ceph osd activate all
         '';
 
         reload = lib.optionalString role.reactivate ''
-          ${fc-ceph}/bin/fc-ceph osd reactivate all
+          ${cephPkgs.fc-ceph}/bin/fc-ceph osd reactivate all
         '';
 
         preStop = ''
-          ${fc-ceph}/bin/fc-ceph osd deactivate all
+          ${cephPkgs.fc-ceph}/bin/fc-ceph osd deactivate all
         '';
 
         serviceConfig = {
@@ -235,10 +235,10 @@ in
           Restart = "always";
           PIDFile = "/run/ceph/osd.%i.pid";
           ExecStart = ''
-            ${fc-ceph}/bin/fc-ceph osd activate --as-systemd-unit  %i
+            ${cephPkgs.fc-ceph}/bin/fc-ceph osd activate --as-systemd-unit  %i
           '';
           ExecStop = ''
-            ${fc-ceph}/bin/fc-ceph osd deactivate --as-systemd-unit %i
+            ${cephPkgs.fc-ceph}/bin/fc-ceph osd deactivate --as-systemd-unit %i
           '';
         };
 

diff --git a/nixos/roles/ceph/rgw.nix b/nixos/roles/ceph/rgw.nix
@@ -14,6 +14,8 @@ let
   rgws = (sort lessThan (map (service: service.address) (fclib.findServices "ceph_rgw-server")));
   first_rgw = if rgws == [ ] then "" else head (lib.splitString "." (head rgws));
 
+  cephPkgs = fclib.ceph.mkPkgs role.cephRelease;
+
   defaultRgwSettings = {
     host = config.networking.hostName;
     keyring = "/etc/ceph/ceph.${username}.keyring";
@@ -107,7 +109,7 @@ in
         serviceConfig = {
           Type = "simple";
           Restart = "always";
-          ExecStart = "${fclib.ceph.releasePkgs.${role.cephRelease}.ceph}/bin/radosgw -n ${username} -f -c /etc/ceph/ceph.conf";
+          ExecStart = "${cephPkgs.ceph}/bin/radosgw -n ${username} -f -c /etc/ceph/ceph.conf";
         };
       };
 
@@ -148,7 +150,7 @@ in
       systemd.services.fc-ceph-rgw-update-stats = {
         description = "Update RGW stats";
         serviceConfig.Type = "oneshot";
-        path = [ fclib.ceph.releasePkgs.${role.cephRelease}.ceph pkgs.jq ];
+        path = [ cephPkgs.ceph pkgs.jq ];
         script = ''
           for uid in $(radosgw-admin metadata list user | jq -r '.[]'); do
             echo $uid