diff --git a/internal_tests/scanning/environment/development.docker b/internal_tests/scanning/environment/development.docker new file mode 100644 index 000000000..da28d908f --- /dev/null +++ b/internal_tests/scanning/environment/development.docker @@ -0,0 +1,10 @@ +# Define the development environment for NCAR project machine seneca +# Based on settings in /usr/local/src/met/README.snat + +# Top-level MET project directory +MET_PROJ_DIR=`ls -1d /met/MET*` + +# SonarQube +export SONARQUBE_DIR=/d1/projects/SonarQube/ +export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86 +export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin diff --git a/internal_tests/scanning/environment/development.seneca b/internal_tests/scanning/environment/development.seneca new file mode 100644 index 000000000..acfbbcf4d --- /dev/null +++ b/internal_tests/scanning/environment/development.seneca @@ -0,0 +1,10 @@ +# Define the development environment for NCAR project machine seneca +# Based on settings in /usr/local/src/met/README.snat + +# Top-level MET project directory +MET_PROJ_DIR=/d1/projects/MET + +# SonarQube +#export SONARQUBE_DIR=/d1/projects/SonarQube/ +#export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86 +#export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin diff --git a/internal_tests/scanning/sornaqube/run_nightly.sh b/internal_tests/scanning/sornaqube/run_nightly.sh new file mode 100755 index 000000000..c8af17253 --- /dev/null +++ b/internal_tests/scanning/sornaqube/run_nightly.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Run nightly SonarQube scan +#======================================================================= +# +# This run_nightly.sh script calls the run_sonarqube.sh script. +# It is intented to be run nightly through cron. Output should be +# directed to the LOGFILE, per cron convention. To run this script, use +# the following commands: +# +# git clone https://github.com/dtcenter/MET +# MET/scripts/sosnarqube/run_nightly.sh name +# +# Usage: run_nightly.sh name +# where "name" specifies a branch, tag, or hash +# +# For example, scan the develop branch: +# run_nightly.sh develop +# +#======================================================================= + +# Constants +#EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu linden@ucar.edu mccabe@ucar.edu" +EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu mccabe@ucar.edu" +#EMAIL_LIST="hsoh@ucar.edu" +KEEP_DAYS=5 + +function usage { + echo + echo "USAGE: run_nightly.sh name" + echo " where \"name\" specifies a branch, tag, or hash." + echo +} + +# Check for arguments +if [ $# -lt 1 ]; then usage; exit 1; fi + +# Store the full path to the scripts directory +SCRIPT_DIR=`dirname $0` +if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi + +# Define the development environment +ENV_FILE=${SCRIPT_DIR}/../environment/development.`hostname` +if [[ ! -e ${ENV_FILE} ]]; then + echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}" + exit 1 +fi +source ${ENV_FILE} + +SONARQUBE_WORK_DIR=${MET_PROJ_DIR}/MET_regression/sonarqube_METplus + +# Delete old directories +find ${SONARQUBE_WORK_DIR} -mtime +${KEEP_DAYS} -name "NB*" | \ + xargs rm -rf + +# Create and switch to a run directory +TODAY=`date +%Y%m%d` +YESTERDAY=`date -d "1 day ago" +%Y%m%d` +RUN_DIR=${SONARQUBE_WORK_DIR}/NB${TODAY} +[[ -e ${RUN_DIR} ]] && rm -rf ${RUN_DIR} +mkdir -p ${RUN_DIR} +cd ${RUN_DIR} + +# Create a logfile +LOGFILE=${RUN_DIR}/run_sonarqube_${TODAY}.log + +# Run scan and check for bad return status +${SCRIPT_DIR}/run_sonarqube.sh ${1} >& ${LOGFILE} +if [[ $? -ne 0 ]]; then + echo "$0: The nightly SonarQube scan for METplus FAILED in `basename ${RUN_DIR}`." >> ${LOGFILE} + cat ${LOGFILE} | mail -s "METplus SonarQube scan Failed for ${1} in `basename ${RUN_DIR}` (autogen msg)" ${EMAIL_LIST} + exit 1 +fi + +# Convert SonarQube report from pdf to html + +exit 0 diff --git a/internal_tests/scanning/sornaqube/run_sonarqube.sh b/internal_tests/scanning/sornaqube/run_sonarqube.sh new file mode 100755 index 000000000..e1786db42 --- /dev/null +++ b/internal_tests/scanning/sornaqube/run_sonarqube.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# +# Run SonarQube Source Code Analyzer on a specified revision of MET +#======================================================================= +# +# This run_sonarqube.sh script will check out the specified version +# of MET and run the SonarQube Source Code Analyzer on it. First, +# go to the directory where you would like the SCA output written and +# then run: +# +# git clone https://github.com/dtcenter/MET +# MET/scripts/sonarqube/run_sonarqube_sca.sh name +# +# Usage: run_sonarqube_sca.sh name +# Test the specified branched version of MET: +# run_sonarqube_sca.sh {branch name} +# Test the specified tagged version of MET: +# run_sonarqube_sca.sh {tag name} +# +#======================================================================= + +# Constants +GIT_REPO="https://github.com/dtcenter/METplus" + +function usage { + echo + echo "USAGE: $(basename $0) name" + echo " where \"name\" specifies a branch, tag, or hash." + echo +} + +# Check for arguments +if [[ $# -lt 1 ]]; then usage; exit; fi + +# Check that SONARQUBE_WRAPPER_BIN is defined +if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then + which build-wrapper-linux-x86-64 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_WRAPPER_BIN=$(which build-wrapper-linux-x86-64 2> /dev/null) + else + which build-wrapper 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_WRAPPER_BIN=$(which build-wrapper 2> /dev/null) + else + echo "ERROR: SONARQUBE_WRAPPER_BIN must be set" + exit 1 + fi + fi +fi +if [ ! -e ${SONARQUBE_WRAPPER_BIN} ]; then + echo "ERROR: SONARQUBE_WRAPPER_BIN (${SONARQUBE_WRAPPER_BIN}) does not exist" + exit 1 +fi + +# Check that SONARQUBE_SCANNER_BIN is defined +if [ -z ${SONARQUBE_SCANNER_BIN} ]; then + which sonar-scanner 2> /dev/null + if [ $? -eq 0 ]; then + SONARQUBE_SCANNER_BIN=$(which sonar-scanner 2> /dev/null) + else + echo "ERROR: SONARQUBE_SCANNER_BIN must be set" + exit 1 + fi +fi +if [ ! -e ${SONARQUBE_SCANNER_BIN} ]; then + echo "ERROR: SONARQUBE_SCANNER_BIN (${SONARQUBE_SCANNER_BIN}) does not exist" + exit 1 +fi + +# Sub-routine for running a command and checking return status +function run_command() { + + # Print the command being called + echo "CALLING: $1" + + # Run the command and store the return status + $1 + STATUS=$? + + # Check return status + if [[ ${STATUS} -ne 0 ]]; then + echo "ERROR: Command returned with non-zero status ($STATUS): $1" + exit ${STATUS} + fi + + return ${STATUS} +} + + +# Store the full path to the scripts directory +SCRIPT_DIR=`dirname $0` +if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi + +# Clone repo into a sub-directory and checkout the requested version +REPO_DIR="METplus-${1}" + +if [ -e ${REPO_DIR} ]; then + run_command "rm -rf ${REPO_DIR}" +fi +run_command "git clone ${GIT_REPO} ${REPO_DIR}" +run_command "cd ${REPO_DIR}" +run_command "git checkout ${1}" + +# Set the build id +#BUILD_ID="MET-${1}" + +SONAR_PROPERTIES=sonar-project.properties + +# Copy sonar-project.properties for Python code +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES + +# Run SonarQube scan for Python code +run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" + +# Run SonarQube report generator to make a PDF file +#TODAY=`date +%Y%m%d` diff --git a/internal_tests/scanning/sornaqube/sonar-project.properties b/internal_tests/scanning/sornaqube/sonar-project.properties new file mode 100644 index 000000000..501fb3051 --- /dev/null +++ b/internal_tests/scanning/sornaqube/sonar-project.properties @@ -0,0 +1,17 @@ +sonar.projectKey=org.sonarqube:METplus_python_NB +sonar.projectName=METplus Nightly Build +sonar.projectVersion=1.0 + +sonar.sources=docs,internal_tests,manage_externals,metplus,parm,produtil,ush + +# The build-wrapper output dir + +# Encoding of the source files +sonar.sourceEncoding=UTF-8 + +#----- Default SonarQube server +#sonar.host.url=http://localhost:9000 +sonar.host.url=http://mandan:9000 + +sonar.login=met +sonar.password=met@sonar.ucar