Add behaviour to remove older backups

CHANGELOG.md

@@ -1,6 +1,12 @@
 # CHANGELOG
 ## Release 124.1.5 (in development)
 
+### Enhancements
+
+- Add cleanup functionality to the backup replication job to
+  only keep the latest 5 backup archives
+  (PR[#4038](https://github.com/scality/metalk8s/pull/4038))
+
 ### Bug fixes
 
 - Make metalk8s-sosreport packages compatible with sos version 4.5
@@ -11,6 +17,9 @@
   regenerated
   (PR[#4035](https://github.com/scality/metalk8s/pull/4035))
 
+- Fix a bug that skips backup archive replication
+  (PR[#4038](https://github.com/scality/metalk8s/pull/4038))
+
 ## Release 124.1.4
 
 ### Bug fixes

salt/metalk8s/orchestrate/backup/files/job.yaml.j2

@@ -22,15 +22,15 @@ spec:
       containers:
       - name: backup-replication
         image: {{ image }}
-        command:
-          - wget
-          - --accept="*.tar.gz"
-          - --no-host-directories
-          - --mirror
-          - --ca-certificate=/certificate/ca.crt
-          - --user=$(BACKUP_USERNAME)
-          - --password=$(BACKUP_PASSWORD)
-          - https://backup/
+        command: ["/bin/sh", "-c"]
+        args:
+          - >-
+            wget --accept="*.tar.gz"
+            --no-host-directories --mirror
+            --ca-certificate=/certificate/ca.crt
+            --user=$(BACKUP_USERNAME)
+            --password=$(BACKUP_PASSWORD)
+            https://backup/; ls -t -1 /backups/*.tar.gz | tail -n+6 | xargs -t rm -f
         env:
           - name: BACKUP_USERNAME
             valueFrom:

tests/post/features/backup.feature

@@ -0,0 +1,5 @@
+@post @ci @local @backup
+Feature: Backup
+    Scenario: Backup multiple times
+        When we run the backup script 6 times
+        Then we have 5 backups on each node

tests/post/steps/test_backup.py

@@ -0,0 +1,62 @@
+import yaml
+
+from pytest_bdd import scenario, when, then, parsers
+
+from tests import utils
+
+# Scenarios
+@scenario("../features/backup.feature", "Backup multiple times")
+def test_backup(host):
+    pass
+
+
+# When
+@when(parsers.parse("we run the backup script {times} times"))
+def run_backup(request, host, times):
+    iso_root = request.config.getoption("--iso-root")
+
+    for _ in range(int(times)):
+        with host.sudo():
+            res = host.run("%s/backup.sh", str(iso_root))
+            assert res.rc == 0, res.stdout
+
+
+# Then
+@then(parsers.parse("we have {count} backups on each node"))
+def check_backup_archive_count(host, version, ssh_config, count):
+    # using yaml becasue json output is not always correctly formatted
+    get_master_nodes_command = [
+        "salt-run",
+        "--out yaml",
+        "salt.cmd",
+        "metalk8s.minions_by_role",
+        "master",
+        "with_pillar=True",
+    ]
+    master_nodes = yaml.load(
+        utils.run_salt_command(host, get_master_nodes_command, ssh_config).stdout,
+        Loader=yaml.Loader,
+    )
+    assert len(master_nodes) > 0, "No master node found"
+
+    if len(master_nodes) == 1:
+        master_nodes = master_nodes[0]
+    else:
+        master_nodes = f"-L {','.join(master_nodes)}"
+
+    command = [
+        "salt",
+        "--out yaml",
+        master_nodes,
+        "cmd.run_all",
+        '"find /var/lib/metalk8s/backups -name "*.tar.gz" | wc -l"',
+    ]
+    backup_counts = yaml.load(
+        utils.run_salt_command(host, command, ssh_config).stdout, Loader=yaml.Loader
+    )
+
+    for name, ret in backup_counts.items():
+        assert ret["retcode"] == 0, ret["stderr"]
+        assert int(ret["stdout"]) == int(count), (
+            f"Expected {count} backup archives on " f'node {name}, got {ret["stdout"]}'
+        )