Merge pull request #1202 from cevich/mirror_skopeo_changes

Cirrus: Improve test synchronization with c/skopeo

.cirrus.yml

@@ -16,12 +16,19 @@ env:
     GOBIN: "${GOPATH}/bin"
     GOCACHE: "${GOPATH}/cache"
     GOSRC: &gosrc "/var/tmp/go/src/github.com/containers/image"
+    # Required for consistency with containers/skopeo CI
+    SKOPEO_PATH: "${GOPATH}/src/github.com/containers/skopeo"
     CIRRUS_WORKING_DIR: *gosrc
     # The default is 'sh' if unspecified
     CIRRUS_SHELL: "/usr/bin/bash"
     # Save a little typing (path relative to $CIRRUS_WORKING_DIR)
     SCRIPT_BASE: "./contrib/cirrus"
 
+    # Normally set to an empty-string.
+    # Temporarily set this to a containers/skopeo PR number to
+    # test changes made there, in a PR in this repository.
+    SKOPEO_PR:
+
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ####
@@ -84,7 +91,6 @@ cross_task:
 test_task:
     alias: test
     depends_on:
-        - cross
         - validate
     container: *std_container
     matrix:
@@ -99,11 +105,19 @@ test_task:
         make test BUILDTAGS="$BUILDTAGS"
 
 
+#####
+###### NOTE: This task is subtantially duplicated in the containers/skopeo
+###### repository's `.cirrus.yml`.  Changes needed here should first be
+###### merged in that repository, before being manually duplicated here.
+######
+###### The *ONLY* exception is in "setup_script" which must initially
+###### clone the skopeo repository.
+######
 test_skopeo_task:
     alias: test_skopeo
     depends_on:
-        - cross
         - validate
+        - cross
     gce_instance:
         image_project: libpod-218412
         zone: "us-central1-f"
@@ -120,53 +134,20 @@ test_skopeo_task:
         - name: "Skopeo w/ opengpg"
           env:
               BUILDTAGS: 'btrfs_noversion libdm_no_deferred_remove containers_image_openpgp'
-    env:
-        SKOPEO_PATH: "${GOPATH}/src/github.com/containers/skopeo"
-        # Normally set to an empty-string.
-        # Temporarily set this to a containers/skopeo PR number to test changes.
-        SKOPEO_PR:
-        # TODO: Remove test technical-debt requiring integration tests to run
-        #       inside a container and with --privileged.  This is hiding bugs!
-        PODMANMAKE: >-
-            podman run -it --rm --privileged
-                -e GOPATH=$GOPATH
-                -v $GOPATH:$GOPATH:Z
-                -w $SKOPEO_PATH
-                $SKOPEO_CI_CONTAINER_FQIN
-                    make
     setup_script: |
-        # This is required as part of the standard Fedora VM setup
-        growpart /dev/sda 1
-        resize2fs /dev/sda1
-
-        # VM's come with the distro. skopeo pre-installed
-        dnf erase -y skopeo
-
-        export "PATH=$PATH:$GOPATH/bin"
-        make tools
-        project_module=$(GO111MODULE="on" go list .)
-        rm -rf "${SKOPEO_PATH}"
-        git clone -b ${SKOPEO_CI_TAG} \
-            https://github.com/containers/skopeo.git ${SKOPEO_PATH}
-        cd "${SKOPEO_PATH}"
-        if [[ -n "$SKOPEO_PR" ]] && [[ $SKOPEO_PR -gt 1000 ]]; then
-            git fetch origin "+refs/pull/$SKOPEO_PR/head"
-            git checkout FETCH_HEAD
-        fi
-        GO111MODULE="on" go mod edit -replace ${project_module}=$GOSRC
-        echo "root:0:1" | tee -a /etc/subuid >> /etc/subgid
-    # Note: We _HAVE_ to spam BUILDTAGS in this way due to how Cirrus-CI
-    # mangles quoted strings when performing in-line 'env' substitution.
-    main_script: |
-        cd "${SKOPEO_PATH}"
-        set -x
-        $PODMANMAKE vendor BUILDTAGS="$BUILDTAGS"
-        $PODMANMAKE bin/skopeo BUILDTAGS="$BUILDTAGS"
-        $PODMANMAKE validate-local BUILDTAGS="$BUILDTAGS"
-        $PODMANMAKE test-unit-local BUILDTAGS="$BUILDTAGS"
-        $PODMANMAKE test-integration-local BUILDTAGS="$BUILDTAGS"
-        # Do not run inside podman, uses containers internally.
-        make test-system-local BUILDTAGS="$BUILDTAGS"
+        "${GOSRC}/${SCRIPT_BASE}/runner.sh" setup
+    vendor_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" vendor
+    build_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" build
+    validate_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" validate
+    unit_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" unit
+    integration_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" integration
+    system_script: >-
+        "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" system
 
 
 # This task is critical.  It updates the "last-used by" timestamp stored

contrib/cirrus/runner.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+# This script is intended to be executed by automation or humans
+# under a hack/get_ci_vm.sh context.  Use under any other circumstances
+# is unlikely to function.
+
+set -e
+
+if [[ -r "/etc/automation_environment" ]]; then
+    source /etc/automation_environment
+    source $AUTOMATION_LIB_PATH/common_lib.sh
+else
+    (
+    echo "WARNING: It does not appear that containers/automation was installed."
+    echo "         Functionality of most of ${BASH_SOURCE[0]} will be negatively"
+    echo "         impacted."
+    ) > /dev/stderr
+fi
+
+OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
+# GCE image-name compatible string representation of distribution _major_ version
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
+# Combined to ease some usage
+OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+
+export "PATH=$PATH:$GOPATH/bin"
+
+_run_setup() {
+    req_env_vars SKOPEO_PATH SKOPEO_CI_TAG GOSRC
+
+    project_module=$(go list .)
+
+    make tools
+
+    rm -rf "${SKOPEO_PATH}"
+    git clone -b ${SKOPEO_CI_TAG} \
+        https://github.com/containers/skopeo.git ${SKOPEO_PATH}
+
+    cd "${SKOPEO_PATH}"
+    if [[ -n "$SKOPEO_PR" ]] && [[ $SKOPEO_PR -gt 1000 ]]; then
+        warn "Fetching and checking out code from skopeo pull-request #$SKOPEO_PR"
+        git fetch origin "+refs/pull/$SKOPEO_PR/head"
+        git checkout FETCH_HEAD
+    fi
+
+    msg "Replacing upstream skopeo $SKOPEO_CI_TAG branch $project_module module"
+    go mod edit -replace ${project_module}=$GOSRC
+
+    "${SKOPEO_PATH}/${SCRIPT_BASE}/runner.sh" setup
+}
+
+req_env_vars GOSRC
+
+handler="_run_${1}"
+if [ "$(type -t $handler)" != "function" ]; then
+    die "Unknown/Unsupported command-line argument '$1'"
+fi
+
+msg "************************************************************"
+msg "Runner executing $1 on $OS_REL_VER"
+msg "************************************************************"
+
+cd "$GOSRC"
+$handler

hack/get_ci_vm.sh

@@ -44,13 +44,8 @@ elif [[ "$1" == "--setup" ]]; then
     in_get_ci_vm "$1"
     # get_ci_vm container entrypoint calls us with this option on the
     # Cirrus-CI environment instance, to perform repo.-specific setup.
-    cd $REPO_DIRPATH
-    echo "+ Configuring environment" > /dev/stderr
-    echo 'PATH=$PATH:$GOPATH/bin' > /etc/ci_environment
-
-    echo "+ Expanding root filesystem" > /dev/stderr
-    growpart /dev/sda 1
-    resize2fs /dev/sda1
+    echo "+ Executing setup" > /dev/stderr
+    ${GOSRC}/${SCRIPT_BASE}/runner.sh setup
 else
     # Create and access VM for specified Cirrus-CI task
     mkdir -p $HOME/.config/gcloud/ssh