Schedule and replicate bootstrap daily backup

CHANGELOG.md

@@ -7,6 +7,12 @@
   MetalK8s volumes
   (PR[#3566](https://github.com/scality/metalk8s/pull/3566))
 
+### Features Added
+
+- A daily backup of the bootstrap node is now automatically scheduled.
+  All the backups are also replicated onto all the master nodes.
+  (PR [#3557](https://github.com/scality/metalk8s/pull/3557))
+
 ## Release 2.10.3
 ### Enhancements
 

buildchain/buildchain/salt_tree.py

@@ -402,6 +402,17 @@ def _get_parts(self) -> Iterator[str]:
     Path("salt/metalk8s/addons/metallb/deployed/config.sls"),
     Path("salt/metalk8s/addons/metallb/deployed/init.sls"),
     Path("salt/metalk8s/addons/metallb/deployed/namespace.sls"),
+    Path("salt/metalk8s/backup/certs/ca.sls"),
+    Path("salt/metalk8s/backup/certs/server.sls"),
+    Path("salt/metalk8s/backup/configured.sls"),
+    Path("salt/metalk8s/backup/deployed/configmap-ca-cert.sls"),
+    Path("salt/metalk8s/backup/deployed/configmap-nginx-config.sls"),
+    Path("salt/metalk8s/backup/deployed/deployment.sls"),
+    Path("salt/metalk8s/backup/deployed/init.sls"),
+    Path("salt/metalk8s/backup/deployed/networkpolicy.sls"),
+    Path("salt/metalk8s/backup/deployed/secret-credentials.sls"),
+    Path("salt/metalk8s/backup/deployed/secret-tls.sls"),
+    Path("salt/metalk8s/backup/deployed/service.sls"),
     Path("salt/metalk8s/beacon/certificates.sls"),
     Path("salt/metalk8s/container-engine/containerd/configured.sls"),
     Path("salt/metalk8s/container-engine/containerd/files/50-metalk8s.conf.j2"),
@@ -511,6 +522,8 @@ def _get_parts(self) -> Iterator[str]:
     Path("salt/metalk8s/orchestrate/deploy_node.sls"),
     Path("salt/metalk8s/orchestrate/etcd.sls"),
     Path("salt/metalk8s/orchestrate/register_etcd.sls"),
+    Path("salt/metalk8s/orchestrate/backup/files/job.yaml.j2"),
+    Path("salt/metalk8s/orchestrate/backup/replication.sls"),
     Path("salt/metalk8s/orchestrate/bootstrap/init.sls"),
     Path("salt/metalk8s/orchestrate/bootstrap/accept-minion.sls"),
     Path("salt/metalk8s/orchestrate/bootstrap/pre-downgrade.sls"),

docs/operation/disaster_recovery/bootstrap_backup_restore.rst

@@ -25,7 +25,7 @@ To create a new backup file, run the following command:
 
     /srv/scality/metalk8s-|version|/backup.sh
 
-Backup archives are stored in /var/lib/metalk8s/.
+Backup archives are stored in /var/lib/metalk8s/backups on all master nodes.
 
 Restoring a Bootstrap Node
 **************************

pillar/metalk8s/roles/bootstrap.sls

@@ -13,6 +13,8 @@ certificates:
         watched: True
   server:
     files:
+      backup-server:
+        watched: True
       control-plane-ingress:
         watched: True
       dex:

pillar/metalk8s/roles/ca.sls

@@ -23,6 +23,10 @@ mine_functions:
     - mine_function: hashutil.base64_encodefile
     - /etc/metalk8s/pki/nginx-ingress/ca.crt
 
+  backup_server_ca_b64:
+    - mine_function: hashutil.base64_encodefile
+    - /etc/metalk8s/pki/backup-server/ca.crt
+
 x509_signing_policies:
   kube_apiserver_client_policy:
     - minions: '*'
@@ -66,3 +70,9 @@ x509_signing_policies:
     - signing_cert: /etc/metalk8s/pki/dex/ca.crt
     - keyUsage: critical digitalSignature, keyEncipherment
     - extendedKeyUsage: serverAuth
+  backup_server_policy:
+    - minions: '*'
+    - signing_private_key: /etc/metalk8s/pki/backup-server/ca.key
+    - signing_cert: /etc/metalk8s/pki/backup-server/ca.crt
+    - keyUsage: critical digitalSignature, keyEncipherment
+    - extendedKeyUsage: serverAuth

salt/_modules/metalk8s.py

@@ -691,3 +691,27 @@ def configure_archive(archive, remove=False):
     msg = "Archive '{0}' {1}".format(archive, msg)
     log.info(msg)
     return msg
+
+
+def backup_node():
+    metalk8s_version = __pillar__["metalk8s"]["cluster_version"]
+    archives = get_archives()
+
+    try:
+        archive_path = archives[f"metalk8s-{metalk8s_version}"]["path"]
+    except KeyError as exc:
+        raise CommandExecutionError(
+            f"No MetalK8s archive found for version {metalk8s_version}"
+        ) from exc
+
+    backup_script = f"{archive_path}/backup.sh"
+    result = __salt__["cmd.run_all"](cmd=backup_script)
+    log.debug("Result: %r", result)
+
+    if result["retcode"] != 0:
+        output = result.get("stderr") or result["stdout"]
+        raise CommandExecutionError(f"Failed to run {backup_script}: {output}")
+
+    msg = "Backup successfully generated"
+    log.info(msg)
+    return msg

salt/_utils/kubernetes_utils.py

@@ -296,13 +296,27 @@ def __init__(self, model, api_cls, name, method_names=None):
             name="api_service",
         ),
         # }}}
+        # /apis/batch/v1/ {{{
+        ("batch/v1", "Job"): KindInfo(
+            model=k8s_client.V1Job,
+            api_cls=k8s_client.BatchV1Api,
+            name="namespaced_job",
+        ),
+        # }}}
         # /apis/batch/v1beta1/ {{{
         ("batch/v1beta1", "CronJob"): KindInfo(
             model=k8s_client.V1beta1CronJob,
             api_cls=k8s_client.BatchV1beta1Api,
             name="namespaced_cron_job",
         ),
         # }}}
+        # /apis/networking.k8s.io/v1/ {{{
+        ("networking.k8s.io/v1", "NetworkPolicy"): KindInfo(
+            model=k8s_client.V1NetworkPolicy,
+            api_cls=k8s_client.NetworkingV1Api,
+            name="namespaced_network_policy",
+        ),
+        # }}}
         # /apis/networking.k8s.io/v1beta1/ {{{
         ("networking.k8s.io/v1beta1", "Ingress"): KindInfo(
             model=k8s_client.NetworkingV1beta1Ingress,

salt/metalk8s/backup/certs/ca.sls

@@ -0,0 +1,37 @@
+{%- from "metalk8s/map.jinja" import backup_server with context %}
+
+{%- set private_key_path = "/etc/metalk8s/pki/backup-server/ca.key" %}
+
+include:
+  - metalk8s.internal.m2crypto
+
+Create backup server CA private key:
+  x509.private_key_managed:
+    - name: {{ private_key_path }}
+    - bits: 4096
+    - verbose: False
+    - user: root
+    - group: root
+    - mode: '0600'
+    - makedirs: True
+    - dir_mode: '0755'
+    - require:
+      - metalk8s_package_manager: Install m2crypto
+    - unless:
+      - test -f "{{ private_key_path }}"
+
+Generate backup server CA certificate:
+  x509.certificate_managed:
+    - name: /etc/metalk8s/pki/backup-server/ca.crt
+    - signing_private_key: {{ private_key_path }}
+    - CN: backup-server-ca
+    - keyUsage: "critical digitalSignature, keyEncipherment, keyCertSign"
+    - basicConstraints: "critical CA:true"
+    - days_valid: {{ backup_server.ca.cert.days_valid }}
+    - user: root
+    - group: root
+    - mode: '0644'
+    - makedirs: True
+    - dir_mode: '0755'
+    - require:
+      - x509: Create backup server CA private key

salt/metalk8s/backup/certs/server.sls

@@ -0,0 +1,50 @@
+{%- from "metalk8s/map.jinja" import backup_server, certificates with context %}
+
+{%- set private_key_path = "/etc/metalk8s/pki/backup-server/server.key" %}
+
+include:
+  - metalk8s.internal.m2crypto
+
+Create backup server private key:
+  x509.private_key_managed:
+    - name: {{ private_key_path }}
+    - bits: 4096
+    - verbose: False
+    - user: root
+    - group: root
+    - mode: '0600'
+    - makedirs: True
+    - dir_mode: '0755'
+    - require:
+      - metalk8s_package_manager: Install m2crypto
+    - unless:
+      - test -f "{{ private_key_path }}"
+
+{%- set certSANs = [
+    'backup',
+    'backup.kube-system',
+    'backup.kube-system.svc',
+    'backup.kube-system.svc.cluster.local',
+] %}
+
+Generate backup server certificate:
+  x509.certificate_managed:
+    - name: {{ certificates.server.files["backup-server"].path }}
+    - public_key: {{ private_key_path }}
+    - ca_server: {{ pillar.metalk8s.ca.minion }}
+    - signing_policy: {{ backup_server.cert.server_signing_policy }}
+    - CN: backup
+    - subjectAltName: "{{ salt['metalk8s.format_san'](certSANs | unique) }}"
+    - days_valid: {{
+        certificates.server.files["backup-server"].days_valid |
+        default(certificates.server.days_valid) }}
+    - days_remaining: {{
+        certificates.server.files["backup-server"].days_remaining |
+        default(certificates.server.days_remaining) }}
+    - user: root
+    - group: root
+    - mode: '0644'
+    - makedirs: True
+    - dir_mode: '0755'
+    - require:
+      - x509: Create backup server private key

salt/metalk8s/backup/configured.sls

@@ -0,0 +1,4 @@
+Schedule daily backup:
+  schedule.present:
+    - function: metalk8s.backup_node
+    - seconds: 86400

salt/metalk8s/backup/deployed/configmap-ca-cert.sls

@@ -0,0 +1,16 @@
+#!jinja | metalk8s_kubernetes
+
+{%- set ca_cert = salt.file.read('/etc/metalk8s/pki/backup-server/ca.crt') %}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: backup-ca-cert
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: backup
+    app.kubernetes.io/part-of: metalk8s
+    app.kubernetes.io/managed-by: salt
+data:
+  ca.crt: |-
+    {{ ca_cert | indent(4, False) }}

salt/metalk8s/backup/deployed/configmap-nginx-config.sls

@@ -0,0 +1,34 @@
+#!jinja | metalk8s_kubernetes
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: backup-nginx-config
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: backup
+    app.kubernetes.io/part-of: metalk8s
+    app.kubernetes.io/managed-by: salt
+data:
+  ssl.conf: |
+    server {
+        listen              443 ssl;
+        server_name         backup;
+        ssl_certificate     /etc/ssl/backup.crt;
+        ssl_certificate_key /etc/ssl/backup.key;
+        ssl_protocols       TLSv1 TLSv1.1 TLSv1.2;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+
+        auth_basic           "Authentication";
+        auth_basic_user_file /etc/nginx/htpasswd/.htpasswd;
+
+        location /livez {
+            auth_basic off;
+            return 200 "I'm alive!";
+        }
+
+        location / {
+            root      /backups;
+            autoindex on;
+        }
+    }

salt/metalk8s/backup/deployed/deployment.sls

@@ -0,0 +1,97 @@
+#!jinja | metalk8s_kubernetes
+
+{%- from "metalk8s/repo/macro.sls" import build_image_name with context %}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: backup
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: backup
+    app.kubernetes.io/part-of: metalk8s
+    app.kubernetes.io/managed-by: salt
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: backup
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: backup
+        app.kubernetes.io/part-of: metalk8s
+        app.kubernetes.io/managed-by: salt
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/bootstrap: ""
+      initContainers:
+        - name: generate-htpasswd
+          command:
+            - htpasswd
+            - -bc
+            - /htpasswd/.htpasswd
+            - $(BACKUP_USERNAME)
+            - $(BACKUP_PASSWORD)
+          image: {{ build_image_name("metalk8s-utils") }}
+          env:
+            - name: BACKUP_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: backup-credentials
+                  key: username
+            - name: BACKUP_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: backup-credentials
+                  key: password
+          volumeMounts:
+            - mountPath: /htpasswd
+              name: htpasswd
+      containers:
+        - name: backup
+          image: {{ build_image_name("nginx") }}
+          ports:
+            - name: https
+              containerPort: 443
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+          readinessProbe:
+            httpGet:
+              path: /livez
+              port: https
+              scheme: HTTPS
+          volumeMounts:
+            - name: backups
+              mountPath: /backups
+            - name: config
+              mountPath: /etc/nginx/conf.d
+            - name: tls
+              mountPath: /etc/ssl
+            - name: htpasswd
+              mountPath: /etc/nginx/htpasswd
+      tolerations:
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/bootstrap
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/infra
+        operator: Exists
+      volumes:
+        - name: backups
+          hostPath:
+            path: /var/lib/metalk8s/backups
+            type: DirectoryOrCreate
+        - name: config
+          configMap:
+            name: backup-nginx-config
+        - name: tls
+          secret:
+            defaultMode: 400
+            secretName: backup-tls
+        - name: htpasswd
+          emptyDir: {}

salt/metalk8s/backup/deployed/init.sls

@@ -0,0 +1,8 @@
+include:
+  - .configmap-ca-cert
+  - .configmap-nginx-config
+  - .secret-tls
+  - .secret-credentials
+  - .networkpolicy
+  - .deployment
+  - .service