Merge pull request #677 from listx/optimize-backups

Optimize backups

infra/gcp/backup_tools/backup_lib.sh

@@ -20,8 +20,8 @@ set -o pipefail
 set -o xtrace
 
 # GCRANE_REF is the commit SHA to use for building the gcrane binary.
-# Known-good commit from 2019-11-15
-export GCRANE_REF="f8574ec722f4dd4e2703689ea2ffe10c2021adc9"
+# Known-good commit from 2020-04-01
+export GCRANE_REF="3d03ed9b1ca2ad5d78d43832e8e46adc31d2b961"
 
 build_gcrane()
 {
@@ -33,32 +33,29 @@ build_gcrane()
     popd
 }
 
-copy_with_date()
+gcrane_copy()
 {
     local source_gcr_repo
     local backup_gcr_repo
-    local timestamp
 
-    if (( $# != 3 )); then
+    if (( $# != 2 )); then
         cat << EOF >&2
-copy_with_date: usage <source_gcr_repo> <backup_gcr_repo> <timestamp>
-e.g. copy_with_date "us.gcr.io/k8s-artifacts-prod" "us.gcr.io/k8s-artifacts-prod-bak" "2019/01/01/00"
+gcrane_copy: usage <source_gcr_repo> <backup_gcr_repo>
+e.g. gcrane_copy "us.gcr.io/k8s-artifacts-prod" "us.gcr.io/k8s-artifacts-prod-bak"
 EOF
         exit 1
     fi
 
     source_gcr_repo="${1}" # "us.gcr.io/k8s-artifacts-prod"
     backup_gcr_repo="${2}" # "us.gcr.io/k8s-artifacts-prod-bak"
-    timestamp="${3}" # "2019/01/01/00"
 
     # Perform backup by copying all images recursively over.
-    "${GCRANE_CHECKOUT_DIR}/cmd/gcrane/gcrane" cp -r "${source_gcr_repo}" "${backup_gcr_repo}/${timestamp}"
+    "${GCRANE_CHECKOUT_DIR}/cmd/gcrane/gcrane" cp -r -j 10 "${source_gcr_repo}" "${backup_gcr_repo}"
 }
 
-check_creds_exist()
+cred_sanity_check()
 {
-    if [[ ! -f "${GOOGLE_APPLICATION_CREDENTIALS}" ]]; then
-        echo >&2 "\$GOOGLE_APPLICATION_CREDENTIALS (${GOOGLE_APPLICATION_CREDENTIALS}) does not exist"
-        exit 1
-    fi
+    # Sanity check
+    gcloud auth list
+    gcloud --quiet auth configure-docker
 }

infra/gcp/backup_tools/backup_prod.sh

@@ -19,12 +19,9 @@
 # Backs up prod registries. This is a thin orchestrator as all the heavy lifting
 # is done by the gcrane binary.
 #
-# This script requires 2 environment variables to be defined:
+# This script requires 1 environment variable to be defined:
 #
 # 1) GOPATH: toplevel path for checking out gcrane's source code.
-#
-# 2) GOOGLE_APPLICATION_CREDENTIALS: path to the service account (JSON file)
-# that has write access to the backup GCRs.
 
 set -o errexit
 set -o nounset
@@ -45,18 +42,13 @@ prod_repos=(
     us.gcr.io/k8s-artifacts-prod
 )
 
-# Check creds exist.
+# Sanity check
+cred_sanity_check
 
-check_creds_exist
 # Build gcrane first.
 build_gcrane
 
-# We use a timestamp of the form YYYY/MM/DD/HH because this makes the backup
-# folders more easily traversable from a human perspective.
-# E.g. 2019/01/01/00 for 2019-01-01 12:00 AM
-timestamp="$(date -u +"%Y/%m/%d/%H")"
-
 # Copy each region to its backup.
 for repo in "${prod_repos[@]}"; do
-    copy_with_date "${repo}" "${repo}-bak" "${timestamp}"
+    gcrane_copy "${repo}" "${repo}-bak"
 done

infra/gcp/backup_tools/backup_test.sh

@@ -18,12 +18,9 @@
 #
 # Tests for backing up prod registries.
 #
-# This script requires 2 environment variables to be defined:
+# This script requires 1 environment variable to be defined:
 #
 # 1) GOPATH: toplevel path for checking out gcrane's source code.
-#
-# 2) GOOGLE_APPLICATION_CREDENTIALS: path to the service account (JSON file)
-# that has write access to the test backup GCRs.
 
 set -o errexit
 set -o nounset
@@ -34,11 +31,11 @@ export GCRANE_CHECKOUT_DIR="${GOPATH}/src/github.com/google/go-containerregistry
 GCRANE="${GCRANE_CHECKOUT_DIR}/cmd/gcrane/gcrane"
 
 export CIP_CHECKOUT_DIR="${GOPATH}/src/sigs.k8s.io/k8s-container-image-promoter"
-CIP_SNAPSHOT_CMD="${CIP_CHECKOUT_DIR}/cip -no-service-account -minimal-snapshot -output-format=CSV -snapshot"
+CIP_SNAPSHOT_CMD="${CIP_CHECKOUT_DIR}/cip -minimal-snapshot -output-format=CSV -snapshot"
 # CIP_REF is the commit SHA to use for building the cip binary (used only for
 # testing; not used by the actual prod backup job).
-# Known-good commit from 2019-10-28
-CIP_REF="1b6a872584d5560c54eb28e5354e5e71dae9d368"
+# Known-good commit from 2020-04-01
+CIP_REF="feb5dc08b2cbfa2c779c4c5d397dad40e669bc84"
 
 SCRIPT_ROOT="$(dirname "$(readlink -f "$0")")"
 # shellcheck disable=SC1090
@@ -67,9 +64,16 @@ clear_test_backup_repo()
     # For added safety, only delete the repo named
     # "gcr.io/k8s-gcr-backup-test-prod-bak".
     repo="${1}"
+    local i
 
+    i=0
     while [[ -n $("${GCRANE}" ls -r "${repo}") ]]; do
         "${GCRANE}" ls -r "${repo}" | xargs -n1 "${GCRANE}" delete || true
+        ((i=i+1))
+		if (( i == 4 )); then
+			echo >&2 "failed to clear ${repo}"
+			return 1
+		fi
     done
 }
 
@@ -118,19 +122,19 @@ declare -A test_repos=(
     #[eu.gcr.io/k8s-gcr-backup-test-prod]=eu.gcr.io/k8s-gcr-backup-test-prod-bak
 )
 
-# Check creds exist.
-check_creds_exist
+# Sanity check.
+cred_sanity_check
 
 # Build dependencies.
 build_gcrane
 build_cip
 
-# Clear test backup repos (k8s-gcr-backup-test-prod-bak). This will make
+# Clear test repos. This will make
 # reading from them simpler.
-# NOTE: We don't bother clearing the test_repos. This is because once the images
-# we should back up are in place, we can just reuse them across subsequent runs.
 for repo in "${!test_repos[@]}"; do
+    # Clear the k8s-gcr-backup-test-prod-bak repo.
     clear_test_backup_repo "${test_repos[$repo]}"
+    # Clear the k8s-gcr-backup-test-prod repo.
     clear_test_backup_repo "${repo}"
 done
 
@@ -146,16 +150,15 @@ for repo in "${!test_repos[@]}"; do
     populate_test_prod_repo "${repo}"
 done
 
-BACKUP_TIMESTAMP="1970/01/01/00"
 # Copy each region to its backup.
 for repo in "${!test_repos[@]}"; do
-    copy_with_date "${repo}" "${test_repos[$repo]}" "${BACKUP_TIMESTAMP}"
+    gcrane_copy "${repo}" "${test_repos[$repo]}"
 done
 
 # Verify backup contents by listing the images.
 for repo in "${!test_repos[@]}"; do
     error_found=0
-    if ! verify_repo "${test_repos[$repo]}/${BACKUP_TIMESTAMP}"; then
+    if ! verify_repo "${test_repos[$repo]}"; then
         error_found=1
     fi
 done