testrunner

#!/bin/bash
set -Exeuo pipefail

# This script is run multiple times in parallel: once for
# each testsuite defined in the YAML file.

THIS_DIR=$(dirname $0)

source $THIS_DIR/utils/common.sh

main() {

    # NB: see the various NBs in the main() of main.

    # We take a single argument; the state dir index to use.
    # But we still expect the global state dir to be in the
    # $PWD.
    state_idx=$1; shift

    state=state/suite-${state_idx}

    [ -d state ] && [ -d $state ]

    # Make sure we update GitHub if we exit due to errexit.
    # We also do a GitHub update on clean exit.
    ensure_err_github_update

    provision_env

    prepare_env

    build_and_test

    fetch_artifacts

    s3_upload

    final_github_update
}

provision_env() {
    if containerized; then
        ensure_teardown_container
        provision_container
    else
        ssh_setup_key
        if clustered; then
            ensure_teardown_cluster
            provision_cluster
        else
            ensure_teardown_node
            provision_node
        fi
    fi
}

provision_container() {
    local image=$(cat $state/parsed/image)

    if containerized; then
        update_github pending "Provisioning container..."
    fi

    # Let's pre-pull the image so that it doesn't count
    # as part of the test timeout.
    if ! sudo docker pull "$image"; then
        update_github error "Could not pull image '$image'."
        exit 0
    fi

    sudo docker run -d \
        --cidfile $state/cid \
        "$image" sleep infinity
}

provision_node() {

    update_github pending "Provisioning host..."

    # substitute node to use in "<name>:<ip>" format
    if [ -n "${RHCI_DEBUG_USE_NODE:-}" ]; then
        echo "${RHCI_DEBUG_USE_NODE%:*}" > $state/host/node_name
        echo "${RHCI_DEBUG_USE_NODE#*:}" > $state/host/node_addr
    else
        $THIS_DIR/provisioner $state $state/parsed/host $state/host
        if [ -f $state/exit ]; then
            # the provisioner encountered a user error and already updated GH
            exit 0
        fi
    fi
}

provision_cluster() {
    local nhosts=$(cat $state/parsed/nhosts)

    update_github pending "Provisioning cluster..."

    seq 0 $((nhosts - 1)) | xargs -P 0 -n 1 -I {} \
        $THIS_DIR/provisioner $state $state/parsed/host-{} $state/host-{}

    if [ -f $state/exit ]; then
        # a provisioner encountered a user error and already updated GH
        exit 0
    fi

    if container_controlled; then
        provision_container
    else
        # make the first host the controller
        ln -s host-0 $state/host
    fi

    local i=0
    while [ $i -lt $nhosts ]; do
        local name=$(cat $state/parsed/host-$i/name)
        local addr=$(cat $state/host-$i/node_addr)
        name=$(sed 's/[.-]/_/g' <<< "$name")
        echo "export RHCI_${name}_IP=${addr}" >> $state/parsed/envs
        i=$((i + 1))
    done
}

prepare_env() {
    local upload_dir=$state/$github_commit.$state_idx.$(date +%s%N)
    echo $upload_dir > $state/upload_dir
    mkdir $upload_dir

    # inject extra repos before installing packages
    if [ -f $state/parsed/rhci-extras.repo ]; then
        envcmd mkdir -p /etc/yum.repos.d
        envcp $state/parsed/rhci-extras.repo /etc/yum.repos.d
    fi

    if [ -f $state/parsed/packages ]; then
        if on_atomic_host; then
            overlay_packages
        else
            install_packages
        fi
    fi

    if clustered; then
        ssh_setup_cluster
    fi

    envcmd mkdir /var/tmp/checkout
    envcp checkouts/$github_repo/. /var/tmp/checkout
}

ssh_setup_key() {
    set +x
    cat > $state/node_key <<< "$os_privkey"
    chmod 0600 $state/node_key
    set -x
}

ssh_setup_cluster() {
    local nhosts=$(cat $state/parsed/nhosts)

    # since the common case is to interact with the various
    # nodes by ssh, let's make sure it's all set up nicely
    # ahead of time

    # let's go through the hosts once to collect keys
    local i=0
    while [ $i -lt $nhosts ]; do
        local name=$(cat $state/parsed/host-$i/name)
        local addr=$(cat $state/host-$i/node_addr)
        ssh-keyscan $addr 2>/dev/null | \
            sed "s/^/$name,/" >> $state/known_hosts
        echo $addr $name >> $state/hosts
        i=$((i + 1))
    done

    # We use a different key than the one used to provision
    # the nodes here, since we don't want to expose the
    # private key of the OpenStack keypair used. NB: not in
    # a state dir; we don't want to regen a key on every
    # run.
    if [ ! -f cluster_keypair/id_rsa ]; then
        # let's just stick with RSA since it's supported on
        # all platforms
        mkdir -p cluster_keypair
        ssh-keygen -t rsa -b 4096 -N "" -f cluster_keypair/id_rsa
    fi

    if container_controlled; then
        # most base images don't have ssh
        if ! envcmd [ -x /bin/ssh ]; then
            env_make_cache
            envcmd yum install -y openssh-clients
        fi
        envcmd mkdir -m 0600 /root/.ssh
        envcp cluster_keypair/id_rsa /root/.ssh
        envcp $state/known_hosts /root/.ssh
        envcp $state/hosts /etc/hosts.append
        envcmd sh -c "cat /etc/hosts.append >> /etc/hosts"
    fi

    vmipssh() {
        ip=$1; shift
        ssh -q -i $state/node_key \
            -o StrictHostKeyChecking=no \
            -o PasswordAuthentication=no \
            -o UserKnownHostsFile=/dev/null \
            root@$ip "$@"
    }

    i=0
    while [ $i -lt $nhosts ]; do
        local name=$(cat $state/parsed/host-$i/name)
        local addr=$(cat $state/host-$i/node_addr)

        # some of these could be redone more cleanly through
        # cloud-init, though the dynamic aspect would
        # probably end up making it look similar
        vmipssh $addr hostnamectl set-hostname $name
        vmipssh $addr "cat >> /etc/hosts" < $state/hosts
        vmipssh $addr "cat >> /root/.ssh/known_hosts" < $state/known_hosts
        vmipssh $addr "cat > /root/.ssh/id_rsa" < cluster_keypair/id_rsa
        vmipssh $addr chmod 0400 /root/.ssh/id_rsa
        vmipssh $addr "cat >> /root/.ssh/authorized_keys" \
            < cluster_keypair/id_rsa.pub
        i=$((i + 1))
    done

    unset -f vmipssh
}

overlay_packages() {
    local upload_dir=$(cat $state/upload_dir)

    local rc=0
    logged_envcmd $upload_dir/setup.log / - - \
        rpm-ostree install "$(cat $state/parsed/packages)" || rc=$?

    if [ $rc != 0 ]; then
        s3_upload
        update_github error "Could not layer packages." "$(cat $state/url)"
        exit 0
    fi

    vmreboot
}

install_packages() {
    local upload_dir=$(cat $state/upload_dir)

    mgr=yum
    if envcmd rpm -q dnf; then
        mgr=dnf
    fi

    env_make_cache

    local rc=0
    logged_envcmd $upload_dir/setup.log / - - \
        $mgr install -y "$(cat $state/parsed/packages)" || rc=$?

    if [ $rc != 0 ]; then
        s3_upload
        update_github error "Could not install packages." "$(cat $state/url)"
        exit 0
    fi
}

env_make_cache() {

    # This is hacky and sad. Preemptively try to refresh
    # yum/dnf cache to work around flaky distro infras.

    local retries=5
    while [ $retries -gt 0 ]; do
        if envcmd yum makecache; then
            break
        fi
        retries=$((retries - 1))
    done
}

run_loop() {
    local timeout=$1; shift
    local logfile=$1; shift
    local workdir=$1; shift
    local testfile=$1; shift
    local envfile=$1; shift

    local max_date=$(($(date +%s) + $timeout))
    while IFS='' read -r line || [[ -n $line ]]; do

        timeout=$(($max_date - $(date +%s)))
        if [ $timeout -le 0 ]; then
            echo "### TIMED OUT" >> $logfile
            rc=137
            break
        fi

        rc=0
        logged_envcmd $logfile $workdir $envfile $timeout "$line" || rc=$?

        if [ $rc != 0 ]; then
            break
        fi

    done < "$testfile"

    return $rc
}

build_and_test() {
    local upload_dir=$(cat $state/upload_dir)
    local timeout=$(cat $state/parsed/timeout)
    local checkout=checkouts/$github_repo
    local rc=0

    if [ -f $state/parsed/build ]; then
        local config_opts=$(cat $state/parsed/build.config_opts)
        local build_opts=$(cat $state/parsed/build.build_opts)
        local install_opts=$(cat $state/parsed/build.install_opts)

        update_github pending "Building..."

        touch $state/build.sh
        if [ ! -f $checkout/configure ]; then
            if [ -f $checkout/autogen.sh ]; then
                echo "NOCONFIGURE=1 ./autogen.sh" >> $state/build.sh
            elif [ -f $checkout/autogen ]; then
                echo "NOCONFIGURE=1 ./autogen" >> $state/build.sh
            fi
        fi

        local njobs=$(envcmd getconf _NPROCESSORS_ONLN)

        echo "./configure $config_opts" >> $state/build.sh
        echo "make all --jobs $njobs $build_opts" >> $state/build.sh
        echo "make install $install_opts" >> $state/build.sh

        local max_date=$(($(date +%s) + $timeout))

        run_loop \
            $timeout \
            $upload_dir/build.log \
            /var/tmp/checkout \
            $state/build.sh \
            $state/parsed/envs || rc=$?

        timeout=$(($max_date - $(date +%s)))
    fi

    if [ $rc = 0 ] && [ -f $state/parsed/tests ]; then

      update_github pending "Running tests..."

      run_loop \
          $timeout \
          $upload_dir/output.log \
          /var/tmp/checkout \
          $state/parsed/tests \
          $state/parsed/envs || rc=$?
    fi

    echo "$rc" > $state/rc
}

fetch_artifacts() {
    local upload_dir=$(cat $state/upload_dir)

    # let's pull back the artifacts
    if [ -f $state/parsed/artifacts ]; then

        mkdir $upload_dir/artifacts

        local fetched_at_least_one=0
        if host_controlled; then
            local node_addr=$(cat $state/host/node_addr)

            while IFS='' read -r artifact || [[ -n $artifact ]]; do
                path="/var/tmp/checkout/$artifact"
                if vmssh [ -e "$path" ]; then
                    vmscp -r "root@$node_addr:$path" $upload_dir/artifacts
                    fetched_at_least_one=1
                fi
            done < $state/parsed/artifacts
        else
            local cid=$(cat $state/cid)
            while IFS='' read -r artifact || [[ -n $artifact ]]; do
                path="/var/tmp/checkout/$artifact"
                if sudo docker exec $cid [ -e "$path" ]; then
                    sudo docker cp "$cid:$path" $upload_dir/artifacts
                    fetched_at_least_one=1
                fi
            done < $state/parsed/artifacts
        fi

        if [ $fetched_at_least_one == 0 ]; then
            rm -rf $upload_dir/artifacts
        fi
    fi
}

s3_upload() {
    local indexer=$(realpath $THIS_DIR/utils/indexer.py)
    local upload_dir=$(cat $state/upload_dir)
    local s3_object=index.html

    # if we just have output.log or build.log, then just link directly to that
    if [ $(find $upload_dir -mindepth 1 | wc -l) = 1 ]; then
        if [ -f $upload_dir/output.log ]; then
            s3_object=output.log
        elif [ -f $upload_dir/build.log ]; then
            s3_object=build.log
        elif [ -f $upload_dir/setup.log ]; then
            s3_object=setup.log
        fi
    fi

    if [ $s3_object = index.html ]; then
        # don't change directory in current session
        ( cd $upload_dir && $indexer )
    fi

    # only actually upload if we're given $s3_prefix
    if [ -n "${s3_prefix:-}" ]; then

        local full_prefix=$s3_prefix/$github_repo/$(basename $upload_dir)

        # upload logs separately so that we can set the MIME type properly
        aws s3 sync --exclude '*.log' \
            $upload_dir s3://$full_prefix
        aws s3 sync --exclude '*' --include '*.log' --content-type text/plain \
            $upload_dir s3://$full_prefix

        # full address we'll use for the final commit status update
        printf "https://s3.amazonaws.com/%s/%s" \
            $full_prefix $s3_object > $state/url
    fi
}

final_github_update() {
    local rc
    local ghstate
    local desc

    rc=$(cat $state/rc)
    if [ $rc == 124 ] || [ $rc == 137 ]; then
        ghstate=failure
        desc="Test timed out and was aborted."
    elif [ $rc != 0 ]; then
        ghstate=failure
        desc="Test failed with rc $rc."
    else
        ghstate=success
        desc="All tests passed"
        if [ -n "${github_pull_id:-}" ] && [ ! -f state/is_merge_sha ]; then
            desc="$desc, but merge commit could not be tested"
        fi
        desc="${desc}."
    fi

    local url=
    if [ -f $state/url ]; then
        url=$(cat $state/url)
    fi

    update_github $ghstate "$desc" "$url"
}

# $1 -- log file
# $2 -- workdir
# $3 -- envfile or -
# $4 -- timeout or -
logged_envcmd() {
    local logfile=$1; shift
    local workdir=$1; shift
    local envfile=$1; shift
    local timeout=$1; shift

    # seed with standard info
    if [ ! -f $logfile ]; then

        echo "### $(date --utc)" > $logfile

        if [ -n "${github_branch:-}" ]; then
            echo "### Branch $github_branch" >> $logfile
        else
            echo -n "### PR #$github_pull_id" >> $logfile
            # NB: is_merge_sha is in the top-level global state dir
            if [ ! -f state/is_merge_sha ]; then
                echo " (WARNING: not merge sha, check for conflicts)" \
                    >> $logfile
            else
                echo >> $logfile
            fi
        fi

        if [ -n "${BUILD_ID:-}" ]; then
            echo "### BUILD_ID $BUILD_ID" >> $logfile
        fi
    fi

    echo '>>>' "$@" >> $logfile

    # we just create a script and run that to make
    # invocation and redirection easier
    echo "set -euo pipefail" > $state/worker.sh
    if [ $envfile != - ] && [ -f $envfile ]; then
        cat $envfile >> $state/worker.sh
    fi
    echo "exec 2>&1" >> $state/worker.sh
    echo "cd $workdir" >> $state/worker.sh
    echo "$@" >> $state/worker.sh

    envcp $state/worker.sh /var/tmp

    local start=$(date +%s)

    rc=0
    timed_envcmd $timeout sh /var/tmp/worker.sh | tee -a $logfile || rc=$?

    local duration=$(($(date +%s) - $start))

    if [ $rc == 0 ]; then
        echo "### COMPLETED IN ${duration}s" >> $logfile
    elif [ $rc == 137 ]; then
        echo "### TIMED OUT AFTER ${duration}s" >> $logfile
    else
        echo "### EXITED WITH CODE $rc AFTER ${duration}s" >> $logfile
    fi

    return $rc
}

# $1 -- timeout or -
timed_envcmd() {
    timeout=$1; shift

    if [[ $timeout == - ]]; then
        timeout=infinity
    fi

    # There's a tricky bit to note here, docker exec and ssh
    # don't handle arguments exactly the same way. SSH will
    # effectively do a 'sh -c' on the passed command, which
    # means that quoting might be an issue.

    if container_controlled; then
        local cid=$(cat $state/cid)
        sudo timeout --signal=KILL $timeout \
            docker exec $cid "$@"
    else
        local node_addr=$(cat $state/host/node_addr)
        timeout --signal=KILL $timeout \
            ssh -q -n -i $state/node_key \
                -o StrictHostKeyChecking=no \
                -o PasswordAuthentication=no \
                -o UserKnownHostsFile=/dev/null \
                root@$node_addr "$@"
    fi
}

envcmd() {
    timed_envcmd infinity "$@"
}

envcp() {
    target=$1; shift
    remote=$1; shift

    # N.B.: docker cp and rsync have almost the same
    # semantics, which is nice. One exception is that docker
    # wants 'dir/.' to signify copying dir contents, whereas
    # rsync is happy with just 'dir/', so always use '/.'.
    # Also, rsync creates nonexistent dirs, whereas docker
    # does not, so explicitly mkdir beforehand.

    if container_controlled; then
        local cid=$(cat $state/cid)
        sudo docker cp $target $cid:$remote
    else
        local node_addr=$(cat $state/host/node_addr)
        rsync --quiet -az --no-owner --no-group \
            -e "ssh -q -i $state/node_key \
                       -o StrictHostKeyChecking=no \
                       -o PasswordAuthentication=no \
                       -o UserKnownHostsFile=/dev/null" \
            $target root@$node_addr:$remote
    fi
}

vmssh() {
    # NB: we use -n because stdin may be in use (e.g. in a
    # bash while read loop)
    ssh -q -n -i $state/node_key \
        -o StrictHostKeyChecking=no \
        -o PasswordAuthentication=no \
        -o UserKnownHostsFile=/dev/null \
        root@$(cat $state/host/node_addr) "$@"
}

vmscp() {
    scp -q -i $state/node_key \
        -o StrictHostKeyChecking=no \
        -o PasswordAuthentication=no \
        -o UserKnownHostsFile=/dev/null "$@"
}

vmreboot() {
    vmssh systemctl reboot || :
    sleep 3 # give time for port to go down
    ssh_wait $(cat $state/host/node_addr) $state/node_key
}

update_github() {
    local context=$(cat $state/parsed/context)
    common_update_github "$context" "$@"
}

ensure_err_github_update() {
    trap "update_github error 'An internal error occurred.'" ERR
}

teardown_node() {
    teardown_node_impl \
        $(cat $state/host/node_name) \
        $(cat $state/host/node_addr)
}

teardown_node_impl() {
    local node_name=$1; shift
    local node_addr=$1; shift

    if [ -z "$node_name" ]; then
        return
    fi

    if [ -n "$node_addr" ] && \
       [ -n "${os_floating_ip_pool:-}" ]; then
        nova floating-ip-disassociate $node_name $node_addr
        nova floating-ip-delete $node_addr
    fi

    nova delete $node_name
}

ensure_teardown_node() {
    if [ -z "${RHCI_DEBUG_NO_TEARDOWN:-}" ]; then
        trap teardown_node EXIT
    fi
}

teardown_container() {
    if [ -f $state/cid ]; then
        sudo docker rm -f $(cat $state/cid)
    fi
}

ensure_teardown_container() {
    if [ -z "${RHCI_DEBUG_NO_TEARDOWN:-}" ]; then
        trap teardown_container EXIT
    fi
}

teardown_cluster() {
    local nhosts=$(cat $state/parsed/nhosts)

    local i=0
    while [ $i -lt $nhosts ]; do
        teardown_node_impl \
            $(cat $state/host-$i/node_name) \
            $(cat $state/host-$i/node_addr)
        i=$((i + 1))
    done

    if container_controlled; then
        teardown_container
    fi
}

ensure_teardown_cluster() {
    if [ -z "${RHCI_DEBUG_NO_TEARDOWN:-}" ]; then
        trap teardown_cluster EXIT
    fi
}

containerized() {
    [ "$(cat $state/parsed/envtype)" = container ]
}

virtualized() {
    [ "$(cat $state/parsed/envtype)" = host ]
}

clustered() {
    [ "$(cat $state/parsed/envtype)" = cluster ]
}

container_controlled() {
    [ "$(cat $state/parsed/controller)" = container ]
}

host_controlled() {
    [ "$(cat $state/parsed/controller)" = host ]
}

on_atomic_host() {
    ! containerized && vmssh test -f /run/ostree-booted
}

main "$@"