release-qualify

#!/bin/bash
#
# Copyright (C) Extensible Service Proxy Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
################################################################################
#

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
. ${ROOT}/script/all-utilities || { echo "Cannot load Bash utilities"; exit 1; }

function usage() {
  [[ -n "${1}" ]] && echo "${1}"
  echo "usage: ${BASH_SOURCE[0]}
  -g <path_to_gsutil>
  -s <commit sha>
  -t <output dir>"
  exit 1
}

GSUTIL=$(which gsutil) || GSUTIL=~/google-cloud-sdk/bin/gsutil
SHA=""
TMP=~/tmp

while getopts :g:s:t: arg; do
  case ${arg} in
    g) GSUTIL="${OPTARG}";;
    s) SHA="${OPTARG}";;
    t) TMP="${OPTARG}";;
    *) usage "Invalid option: -${OPTARG}";;
  esac
done

[[ -n "${SHA}" ]] || usage "Must provide commit sha via '-s' parameter."
[[ -x "${GSUTIL}" ]] || usage "Cannot find gsutil, provide it via '-g' flag."

mkdir -p "${TMP}"
LOGS="$(mktemp -d ${TMP}/qualify-XXXX)"
RESULT="${LOGS}/RESULT.log"
NGINX_LOG_ANALYSIS="${LOGS}/nginx-log-analysis.log"

function check_result() {
  local FILE=${1}
  local COMMIT=${2}

  python - <<EOF ${FILE} ${COMMIT}
import sys
import json

log_file = sys.argv[1]
commit_sha = sys.argv[2]

with open(log_file) as log:
  result = json.load(log)

json_status = result.get('scriptStatus', 1)
json_sha = result.get('headCommitHash', '')

status_ok = json_status == 0
sha_ok = json_sha == commit_sha

print "Checking {} SHA={}".format(log_file, commit_sha)
print "  Status == {} ({})".format(json_status, "OK" if status_ok else "FAIL")
print "  SHA == {} ({})".format(json_sha, "OK" if sha_ok else "FAIL")

exit(0 if status_ok and sha_ok else 1)
EOF
}

function count_stress_failures() {
  awk '
    BEGIN {
      failed=0
      complete=0
      non2xx=0
    }

    /Complete requests *[0-9]*/ {
      complete+=$3
    }

    /^Failed requests *[0-9]*/ {
      failed+=$3
    }

    /^Non-2xx responses *[0-9]*/ {
      non2xx+=$3
    }

    END {
      total = complete + failed
      print "Failed requests:   ", failed
      print "Non-2xx responses: ", non2xx
      print "Total requests:    ", total
      if (total > 0) {
        print "Failed/Total:      ", (failed + non2xx) / total
      }
    }' "${@}"
}

(
  echo "Release qualification of ${SHA}."
  echo "It is now: $(date)"

  mkdir -p "${LOGS}/${SHA}"

  echo "Downloading Jenkins logs to '${LOGS}' directory."
  ${GSUTIL} -m -q cp -r "gs://endpoints-jenkins.appspot.com/${SHA}/logs/*" "${LOGS}/${SHA}/" 2>&1 \
    || error_exit "Failed to download logs from endpoints-jenkins.appspot.com."

  python "${ROOT}/script/validate_release.py" \
    --commit_sha "${SHA}" \
    --path "${LOGS}/${SHA}" \
    || error_exit "Release is not qualified."

  RQ_TESTS=()

  # This while loop reads from a redirect set up at the "done" clause.
  # Because we read user input inside the loop, we set up the input
  # coming from the "find" command on file descriptor 3. This is why
  # we use "read -u3" here.
  while read -u3 LOG_FILE; do
    DIR="$(dirname "${LOG_FILE}")"
    JSON_FILE="${LOG_FILE%.log}.json"
    RUN="${DIR##*/}"


    [[ -f "${JSON_FILE}" ]] \
      || error_exit "Result of release qualification test ${JSON_FILE} not found."

    echo '*********************************************************************'
    echo "Release qualification run: ${RUN}"
    echo "Backend application was:"
    grep 'PROJECT_ID:\|IMAGE_NAME:\|GAE_VERSION:\|Deployed URL:' \
      "${LOG_FILE}"

    echo ''
    echo "Checking ${JSON_FILE}"
    echo ''
    check_result "${JSON_FILE}" "${SHA}" || continue

    echo ''
    echo "Checking ${LOG_FILE}"
    echo ''
    count_stress_failures "${LOG_FILE}"

    RQ_TESTS+=(${DIR})

  # the ! -path ... exludes the root directory which is otherwise
  # included in the result
  done 3< <(find "${LOGS}/${SHA}" ! -path "${LOGS}/${SHA}" -type f -name 'long-run-test*.log')

  if [[ ${#RQ_TESTS[@]} -le 0 ]]; then
    echo '*********************************************************************'
    echo '* Release qualification INCOMPLETE.                                 *'
    echo '*                       **********                                  *'
    echo '*                                                                   *'
    echo '* No release qualification tests have been run yet.                 *'
    echo '* Please run release qualification tests per http://go/esp-r.       *'
    echo '* When complete, re-run the command below.                          *'
    echo '*********************************************************************'

    ARGS=()
    for arg in "${@}"; do ARGS+=("\"${arg}\""); done
    echo "${BASH_SOURCE[0]}" "${ARGS[@]}"

    exit 0
  fi

cat > "${NGINX_LOG_ANALYSIS}" << __EOF__
**********************************************************************
* Below are the NGINX error log entries that were not accounted for. *
**********************************************************************
__EOF__

  # Read all entries from "${ROOT}/script/known-error-log-entries" and construct
  # a regular expression that would match any of the entries.
  while read entry; do
    [[ "${KNOWN_ERRORS_REGEX}" != '' ]] && KNOWN_ERRORS_REGEX+='\|'
    KNOWN_ERRORS_REGEX+="\($entry\)"
  done<"${ROOT}/script/known-error-log-entries"

  for DIR in ${RQ_TESTS[@]}; do

    echo ""
    echo "Looking for tar.gz files in ${DIR}"
    echo ""
    while read -u3 TAR; do
      echo "Extracting nginx logs from ${TAR}"
      tar zxvf ${TAR} -C "${DIR}" --strip=3 var/log/nginx \
        || tar xvf ${TAR} -C "${DIR}" --strip=3 var/log/nginx
    done 3< <(find "${DIR}" -type f -name '*.tar.gz')

    echo ""
    echo "Looking for error_stderr.log.gz file in ${DIR}"
    echo ""
    if [[ -f "${DIR}/error_stderr.log.gz" ]]; then
      echo "Extracting NGINX logs from ${DIR}/error_stderr.log.gz"
      gunzip -f "${DIR}/error_stderr.log.gz" \
        || error_exit "Failed to gunzip NGINX logs"
    fi

    echo ''
    echo 'Checking for presence of error.log'
    if ls "${DIR}"/error*.log; then
      echo ''
      echo "Analyzing NGINX error logs in ${DIR} ..."
      cat >> "${NGINX_LOG_ANALYSIS}" << __EOF__

**********************************************************************
DIR: ${DIR}
**********************************************************************
__EOF__
      grep -v "${KNOWN_ERRORS_REGEX}" "${DIR}"/error*.log \
        >> "${NGINX_LOG_ANALYSIS}"
    fi
  done;

  printf "\e[31m
Please review the NGINX log analysis results in ${NGINX_LOG_ANALYSIS}
\e[0m"

  echo ''
  echo '*********************************************************************'
  echo '* Release qualification script completed.                           *'
  echo '*                                                                   *'
  echo '* Additional manual checks may be required.                         *'
  echo '*                                                                   *'
  echo '* Please review the results above and analyze any failed requests.  *'
  echo '* If there are failures, review the NGINX error logs on the backend *'
  echo '* virtual machines to get more insights into the failures.          *'
  echo '* Update the release bug with any findings, and open bugs for any   *'
  echo '* issues found during the investigation.                            *'
  echo '*********************************************************************'

) | tee ${RESULT}

[[ ${PIPESTATUS[0]} -eq 0 && ${PIPESTATUS[1]} -eq 0 ]] \
  || error_exit "Release qualification failed. Results were saved in ${RESULT}."

echo "Results were saved in ${RESULT}"

printf '\e[31m
***************************************************************************
*      Please paste the script output verbatim into the release bug.      *
***************************************************************************
\e[0m'