Merge pull request #311 from barrucadu/refactor/sync

Refactor: move bookdb & bookmarks sync scripts into modules
barrucadu · Dec 10, 2024 · 2da8cf3 · 2da8cf3
2 parents 8ed4c55 + c75e388
commit 2da8cf3
Show file tree

Hide file tree

Showing 10 changed files with 301 additions and 118 deletions.
diff --git a/hosts/carcosa/configuration.nix b/hosts/carcosa/configuration.nix
@@ -420,50 +420,13 @@ in
   ## Nyarlathotep Sync
   ###############################################################################
 
-  users.extraUsers.nyarlathotep-remote-sync = {
-    home = "/var/lib/nyarlathotep-remote-sync";
-    createHome = true;
-    isSystemUser = true;
-    openssh.authorizedKeys.keys =
-      [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];
-    shell = pkgs.bashInteractive;
-    group = "nogroup";
-    packages =
-      let
-        bookdb-receive-covers = ''
-          if [[ ! -d ~/bookdb-covers ]]; then
-            echo "bookdb-covers does not exist"
-            exit 1
-          fi
-
-          /run/wrappers/bin/sudo ${pkgs.rsync}/bin/rsync -a --delete ~/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
-          /run/wrappers/bin/sudo ${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
-        '';
-        bookdb-receive-elasticsearch = ''
-          env ES_HOST=${config.systemd.services.bookdb.environment.ES_HOST} \
-              ${pkgs.nixfiles.bookdb}/bin/bookdb_ctl import-index --drop-existing
-        '';
-        bookmarks-receive-elasticsearch = ''
-          env ES_HOST=${config.systemd.services.bookmarks.environment.ES_HOST} \
-              ${pkgs.nixfiles.bookmarks}/bin/bookmarks_ctl import-index --drop-existing
-        '';
-      in
-      [
-        (pkgs.writeShellScriptBin "bookdb-receive-covers" bookdb-receive-covers)
-        (pkgs.writeShellScriptBin "bookdb-receive-elasticsearch" bookdb-receive-elasticsearch)
-        (pkgs.writeShellScriptBin "bookmarks-receive-elasticsearch" bookmarks-receive-elasticsearch)
-      ];
-  };
+  nixfiles.bookdb.remoteSync.receive.enable = true;
+  nixfiles.bookdb.remoteSync.receive.authorizedKeys =
+    [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];
 
-  security.sudo.extraRules = [
-    {
-      users = [ config.users.extraUsers.nyarlathotep-remote-sync.name ];
-      commands = [
-        { command = "${pkgs.rsync}/bin/rsync -a --delete ${config.users.extraUsers.nyarlathotep-remote-sync.home}/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
-        { command = "${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
-      ];
-    }
-  ];
+  nixfiles.bookmarks.remoteSync.receive.enable = true;
+  nixfiles.bookmarks.remoteSync.receive.authorizedKeys =
+    [ "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIChVw9DPLafA3lCLCI4Df9rYuxedFQTXAwDOOHUfZ0Ac remote-sync@nyarlathotep" ];
 
   ###############################################################################
   ## Miscellaneous

diff --git a/hosts/nyarlathotep/configuration.nix b/hosts/nyarlathotep/configuration.nix
@@ -491,85 +491,23 @@ in
   # Remote Sync
   ###############################################################################
 
-  users.extraUsers.remote-sync = {
-    home = "/var/lib/remote-sync";
-    createHome = true;
-    isSystemUser = true;
-    shell = pkgs.bashInteractive;
-    group = "nogroup";
-  };
+  nixfiles.bookdb.remoteSync.send.enable = true;
+  nixfiles.bookdb.remoteSync.send.sshKeyFile = config.sops.secrets."users/bookdb_remote_sync/ssh_private_key".path;
+  nixfiles.bookdb.remoteSync.send.targets = [ "carcosa.barrucadu.co.uk" ];
 
-  systemd.services.bookdb-sync = {
-    description = "Upload bookdb data to carcosa";
-    startAt = "*:15";
-    path = with pkgs; [ openssh rsync ];
-    serviceConfig = {
-      ExecStart = pkgs.writeShellScript "bookdb-sync" ''
-        set -ex
-
-        /run/wrappers/bin/sudo ${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ~/bookdb-covers
-        trap "/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/rm -rf ~/bookdb-covers" EXIT
-        rsync -az\
-              -e "ssh -i $SSH_KEY_FILE -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
-              ~/bookdb-covers/ \
-              [email protected]:~/bookdb-covers/
-        ssh -i "$SSH_KEY_FILE" \
-            -o UserKnownHostsFile=/dev/null \
-            -o StrictHostKeyChecking=no \
-            [email protected] \
-            bookdb-receive-covers
-
-        env "ES_HOST=$ES_HOST" \
-            ${pkgs.nixfiles.bookdb}/bin/bookdb_ctl export-index | \
-        ssh -i "$SSH_KEY_FILE" \
-            -o UserKnownHostsFile=/dev/null \
-            -o StrictHostKeyChecking=no \
-            [email protected] \
-            bookdb-receive-elasticsearch
-      '';
-      User = config.users.extraUsers.remote-sync.name;
-    };
-    environment = {
-      ES_HOST = config.systemd.services.bookdb.environment.ES_HOST;
-      SSH_KEY_FILE = config.sops.secrets."users/remote_sync/ssh_private_key".path;
-    };
+  sops.secrets."users/bookdb_remote_sync/ssh_private_key" = {
+    owner = config.users.extraUsers.bookdb-remote-sync-send.name;
+    key = "users/remote_sync/ssh_private_key";
   };
 
-  systemd.services.bookmarks-sync = {
-    description = "Upload bookmarks data to carcosa";
-    startAt = "*:15";
-    path = with pkgs; [ openssh ];
-    serviceConfig = {
-      ExecStart = pkgs.writeShellScript "bookmarks-sync" ''
-        set -ex
-
-        env "ES_HOST=$ES_HOST" \
-            ${pkgs.nixfiles.bookmarks}/bin/bookmarks_ctl export-index | \
-        ssh -i "$SSH_KEY_FILE" \
-            -o UserKnownHostsFile=/dev/null \
-            -o StrictHostKeyChecking=no \
-            [email protected] \
-            bookmarks-receive-elasticsearch
-      '';
-      User = config.users.extraUsers.remote-sync.name;
-    };
-    environment = {
-      ES_HOST = config.systemd.services.bookmarks.environment.ES_HOST;
-      SSH_KEY_FILE = config.sops.secrets."users/remote_sync/ssh_private_key".path;
-    };
-  };
+  nixfiles.bookmarks.remoteSync.send.enable = true;
+  nixfiles.bookmarks.remoteSync.send.sshKeyFile = config.sops.secrets."users/bookmarks_remote_sync/ssh_private_key".path;
+  nixfiles.bookmarks.remoteSync.send.targets = [ "carcosa.barrucadu.co.uk" ];
 
-  security.sudo.extraRules = [
-    {
-      users = [ config.users.extraUsers.remote-sync.name ];
-      commands = [
-        { command = "${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ${config.users.extraUsers.remote-sync.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
-        { command = "${pkgs.coreutils}/bin/rm -rf ${config.users.extraUsers.remote-sync.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
-      ];
-    }
-  ];
-
-  sops.secrets."users/remote_sync/ssh_private_key".owner = config.users.extraUsers.remote-sync.name;
+  sops.secrets."users/bookmarks_remote_sync/ssh_private_key" = {
+    owner = config.users.extraUsers.bookmarks-remote-sync-send.name;
+    key = "users/remote_sync/ssh_private_key";
+  };
 
   ###############################################################################
   # RSS-to-Mastodon

diff --git a/shared/bookdb/default.nix b/shared/bookdb/default.nix
@@ -21,6 +21,8 @@ in
   imports = [
     ./erase-your-darlings.nix
     ./options.nix
+    ./remote-sync-receive.nix
+    ./remote-sync-send.nix
   ];
 
   config = mkIf cfg.enable {

diff --git a/shared/bookdb/options.nix b/shared/bookdb/options.nix
@@ -71,5 +71,47 @@ with lib;
         Format of the log messages.
       '';
     };
+
+    remoteSync = {
+      receive = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = mdDoc ''
+            Enable receiving push-based remote sync from other hosts.
+          '';
+        };
+        authorizedKeys = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = mdDoc ''
+            SSH public keys to allow pushes from.
+          '';
+        };
+      };
+
+      send = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = mdDoc ''
+            Enable periodically pushing local state to other hosts.
+          '';
+        };
+        sshKeyFile = mkOption {
+          type = types.str;
+          description = mdDoc ''
+            Path to SSH private key.
+          '';
+        };
+        targets = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = mdDoc ''
+            Hosts to push to.
+          '';
+        };
+      };
+    };
   };
 }
diff --git a/shared/bookdb/remote-sync-receive.nix b/shared/bookdb/remote-sync-receive.nix
@@ -0,0 +1,49 @@
+# See remote-sync-send.nix
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.nixfiles.bookdb.remoteSync.receive;
+in
+{
+  config = mkIf cfg.enable {
+    users.extraUsers.bookdb-remote-sync-receive = {
+      home = "/var/lib/bookdb-remote-sync-receive";
+      createHome = true;
+      isSystemUser = true;
+      openssh.authorizedKeys.keys = cfg.authorizedKeys;
+      shell = pkgs.bashInteractive;
+      group = "nogroup";
+      packages =
+        let
+          receive-covers = ''
+            if [[ ! -d ~/bookdb-covers ]]; then
+              echo "bookdb-covers does not exist"
+              exit 1
+            fi
+
+            /run/wrappers/bin/sudo ${pkgs.rsync}/bin/rsync -a --delete ~/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
+            /run/wrappers/bin/sudo ${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR} || exit 1
+          '';
+          receive-elasticsearch = ''
+            env ES_HOST=${config.systemd.services.bookdb.environment.ES_HOST} \
+                ${pkgs.nixfiles.bookdb}/bin/bookdb_ctl import-index --drop-existing
+          '';
+        in
+        [
+          (pkgs.writeShellScriptBin "receive-covers" receive-covers)
+          (pkgs.writeShellScriptBin "receive-elasticsearch" receive-elasticsearch)
+        ];
+    };
+
+    security.sudo.extraRules = [
+      {
+        users = [ config.users.extraUsers.bookdb-remote-sync-receive.name ];
+        commands = [
+          { command = "${pkgs.rsync}/bin/rsync -a --delete ${config.users.extraUsers.bookdb-remote-sync-receive.home}/bookdb-covers/ ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
+          { command = "${pkgs.coreutils}/bin/chown -R ${config.users.users.bookdb.name}.nogroup ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}"; options = [ "NOPASSWD" ]; }
+        ];
+      }
+    ];
+  };
+}
diff --git a/shared/bookdb/remote-sync-send.nix b/shared/bookdb/remote-sync-send.nix
@@ -0,0 +1,69 @@
+# See remote-sync-receive.nix
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.nixfiles.bookdb.remoteSync.send;
+
+  toService = target: {
+    name = "bookdb-sync-${target}";
+    value = {
+      description = "Upload bookdb data to ${target}";
+      startAt = "*:15";
+      path = with pkgs; [ openssh rsync ];
+      serviceConfig = {
+        ExecStart = pkgs.writeShellScript "bookdb-sync" ''
+          set -ex
+
+          /run/wrappers/bin/sudo ${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ~/bookdb-covers
+          trap "/run/wrappers/bin/sudo ${pkgs.coreutils}/bin/rm -rf ~/bookdb-covers" EXIT
+          rsync -az\
+                -e "ssh -i $SSH_KEY_FILE -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
+                ~/bookdb-covers/ \
+                bookdb-remote-sync-receive@${target}:~/bookdb-covers/
+          ssh -i "$SSH_KEY_FILE" \
+              -o UserKnownHostsFile=/dev/null \
+              -o StrictHostKeyChecking=no \
+              bookdb-remote-sync-receive@${target} \
+              receive-covers
+
+          env "ES_HOST=$ES_HOST" \
+              ${pkgs.nixfiles.bookdb}/bin/bookdb_ctl export-index | \
+          ssh -i "$SSH_KEY_FILE" \
+              -o UserKnownHostsFile=/dev/null \
+              -o StrictHostKeyChecking=no \
+              bookdb-remote-sync-receive@${target} \
+              receive-elasticsearch
+        '';
+        User = config.users.extraUsers.bookdb-remote-sync-send.name;
+      };
+      environment = {
+        ES_HOST = config.systemd.services.bookdb.environment.ES_HOST;
+        SSH_KEY_FILE = cfg.sshKeyFile;
+      };
+    };
+  };
+in
+{
+  config = mkIf cfg.enable {
+    users.extraUsers.bookdb-remote-sync-send = {
+      home = "/var/lib/bookdb-remote-sync-send";
+      createHome = true;
+      isSystemUser = true;
+      shell = pkgs.bashInteractive;
+      group = "nogroup";
+    };
+
+    systemd.services = listToAttrs (map toService cfg.targets);
+
+    security.sudo.extraRules = [
+      {
+        users = [ config.users.extraUsers.bookdb-remote-sync-send.name ];
+        commands = [
+          { command = "${pkgs.coreutils}/bin/cp -r ${config.systemd.services.bookdb.environment.BOOKDB_UPLOADS_DIR}/ ${config.users.extraUsers.bookdb-remote-sync-send.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
+          { command = "${pkgs.coreutils}/bin/rm -rf ${config.users.extraUsers.bookdb-remote-sync-send.home}/bookdb-covers"; options = [ "NOPASSWD" ]; }
+        ];
+      }
+    ];
+  };
+}
diff --git a/shared/bookmarks/default.nix b/shared/bookmarks/default.nix
@@ -17,6 +17,8 @@ in
 {
   imports = [
     ./options.nix
+    ./remote-sync-receive.nix
+    ./remote-sync-send.nix
   ];
 
   config = mkIf cfg.enable {

diff --git a/shared/bookmarks/options.nix b/shared/bookmarks/options.nix
@@ -60,5 +60,47 @@ with lib;
         Format of the log messages.
       '';
     };
+
+    remoteSync = {
+      receive = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = mdDoc ''
+            Enable receiving push-based remote sync from other hosts.
+          '';
+        };
+        authorizedKeys = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = mdDoc ''
+            SSH public keys to allow pushes from.
+          '';
+        };
+      };
+
+      send = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = mdDoc ''
+            Enable periodically pushing local state to other hosts.
+          '';
+        };
+        sshKeyFile = mkOption {
+          type = types.str;
+          description = mdDoc ''
+            Path to SSH private key.
+          '';
+        };
+        targets = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = mdDoc ''
+            Hosts to push to.
+          '';
+        };
+      };
+    };
   };
 }