Do backups asynchronously

This takes us to a single point of truth for data; the other is
clearly secondary.

Jenkinsfile.backup

@@ -0,0 +1,45 @@
+def TIMER = "H/30 * * * *"
+def NODE = "atomic-jslave-autobrew"
+
+def dirs_to_backup = ["rdgo", "images" ];
+
+node(NODE) {
+    def par_stages = [:]
+    stage("Clean workspace") {
+       step([$class: 'WsCleanup'])
+    }
+    checkout scm
+    utils = load("pipeline-utils.groovy")
+    utils.define_properties(TIMER)
+
+    for (dir in dirs_to_backup) {
+        // Avoid iteration+closure trap
+        def basetarget = dir;
+        par_stages["backup-" + basetarget] = { -> stage("Backup " + basetarget) {
+    utils.inside_assembler_container("") {
+        def remote = "/srv/rhcos/output/${basetarget}";
+        def local = "${WORKSPACE}/${basetarget}";
+        stage("Sync In") {
+            withCredentials([
+                string(credentialsId: params.ARTIFACT_SERVER, variable: 'ARTIFACT_SERVER'),
+                sshUserPrivateKey(credentialsId: params.ARTIFACT_SSH_CREDS_ID, keyFileVariable: 'KEY_FILE'),
+            ]) {
+                sh "mkdir -p ${local}"
+                utils.rsync_dir_in_dest(ARTIFACT_SERVER, KEY_FILE, remote, local)
+            }
+        }
+
+        stage("Sync Out") {
+            withCredentials([
+                [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: params.AWS_CREDENTIALS],
+            ]) {
+                sh "aws s3 sync --delete ${local}/ s3://rhcos/${basetarget}"
+            }
+        }
+    }
+    } } }
+
+    // Execute
+    parallel par_stages; par_stages = [:]
+}
+

Jenkinsfile.cloud

@@ -229,16 +229,6 @@ node(NODE) {
             utils.rsync_dir_out(ARTIFACT_SERVER, KEY_FILE, images)
         }
     } }
-    par_stages["s3-upload"] = { -> stage("S3 upload") {
-            withCredentials([
-                [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: params.AWS_CREDENTIALS],
-                string(credentialsId: params.S3_PRIVATE_BUCKET, variable: 'S3_PRIVATE_BUCKET'),
-                string(credentialsId: params.AWS_CI_ACCOUNT, variable: 'AWS_CI_ACCOUNT'),
-            ]) { sh """
-                aws s3 sync --delete ${images}/ s3://rhcos/images/
-            """ }
-        }
-    }
     parallel par_stages; par_stages = [:]
     }
 }

Jenkinsfile.rdgo

@@ -71,11 +71,6 @@ node(NODE) {
             ]) {
                 utils.rsync_dir_out(ARTIFACT_SERVER, KEY_FILE, rdgo)
             }
-            withCredentials([
-                [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: params.AWS_CREDENTIALS],
-            ]) {
-                sh "aws s3 sync --delete ${rdgo}/build/ s3://rhcos/rdgo"
-            }
             currentBuild.description = 'rdgo build+sync done'
         }
 

pipeline-utils.groovy

@@ -76,6 +76,14 @@ def rsync_dir_out(server, key, dir) {
     rsync_dir(key, dir, "${server}:${dir}")
 }
 
+def rsync_dir_in_dest(server, key, srcdir, destdir) {
+    rsync_dir(key, "${server}:${srcdir}", destdir)
+}
+
+def rsync_dir_out_dest(server, key, srcdir, destdir) {
+    rsync_dir(key, srcdir, "${server}:${destdir}")
+}
+
 def rsync_dir(key, from_dir, to_dir) {
     sh """
         rsync -Hrlpt --stats --delete --delete-after \