diff --git a/.cmake-format.py b/.cmake-format.py index 230f70bbf..1d8650d3e 100644 --- a/.cmake-format.py +++ b/.cmake-format.py @@ -10,10 +10,10 @@ "add_component_library": { "pargs": 1, - "flags": ["NOEXPORT"], - "kwargs": {"TARGET": '*', "LIBRARY_NAME": '*', "SOURCES": '*', "HEADERS": '*', - "INCLUDE_DIRS": '*', "TARGET_DEPS": '*', "LINK_LANGUAGE": '*', - "LIBRARY_TYPE": '*'}}, + "flags": ["NOEXPORT", "${COMPILE_WITH_CUDA}"], + "kwargs": {"EXPORT_NAME": '*', "HEADERS": '*', "INCLUDE_DIRS": '*', "LIBRARY_NAME": '*', + "LIBRARY_NAME_PREFIX": '*', "LIBRARY_TYPE": '*', "LINK_LANGUAGE": '*', + "SOURCES": '*', "TARGET": '*', "TARGET_DEPS": '*'}}, "add_component_executable": { "kwargs": {"TARGET": '*', "LIBRARY_NAME": '*', "SOURCES": '*', "HEADERS": '*', @@ -25,8 +25,9 @@ "add_parallel_tests": { "flags": ["MPI_PLUS_OMP"], - "kwargs": {"SOURCES": '*', "DEPS": '*', "TEST_ARGS": '*', "PASS_REGEX": '*', - "FAIL_REGEX": '*', "LABEL": '*', "PE_LIST": '*'}}, + "kwargs": {"DEPS": '*', "FAIL_REGEX": '*', "LABEL": '*', "PASS_REGEX": '*', "PE_LIST": + '*', "RESOURCE_LOCK": '*', "RUN_AFTER": '*', "SOURCES": '*', "TEST_ARGS": + '*'}}, "add_app_unit_test": { "kwargs": {"DRIVER": '*', "APPS": '*', "TEST_ARGS": '*', "LABELS": '*', "APP": '*', diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 000000000..8cde95e21 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,27 @@ +#--------------------------------------------------------------------------------------------------# +# File: .gitlab-ci.yml +# Author: Kelly Thompson +# Date: Tuesday, Dec 04, 2018, 15:13 pm +# Purpose: Instructions for Travis continuous integration testing on GitLab. This script wil be run +# to test pull requests +# Note: Copyright (C) 2018-2021 Triad National Security, LLC. All rights reserved. +#--------------------------------------------------------------------------------------------------# + +variables: + PROJECT: Draco + +stages: + - fasttest + - test + - slowtest + - optional + +include: + - local: /.gitlab/ci/ccscs-jobs.yml +# - local: /.gitlab/ci/darwin-jobs.yml + - local: /.gitlab/ci/superlinter.yml +# - local: /.gitlab/ci/msvc-jobs.yml + +#--------------------------------------------------------------------------------------------------# +# End .gitlab-ci.yml +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/README.ci b/.gitlab/ci/README.ci new file mode 100644 index 000000000..a715970f3 --- /dev/null +++ b/.gitlab/ci/README.ci @@ -0,0 +1,8 @@ + +#stages: +#- prepare +#- style +#- autodoc +#- build +#- package +#- release diff --git a/.gitlab/ci/ccscs-jobs.yml b/.gitlab/ci/ccscs-jobs.yml new file mode 100644 index 000000000..90fb685fe --- /dev/null +++ b/.gitlab/ci/ccscs-jobs.yml @@ -0,0 +1,251 @@ +#--------------------------------------------------------------------------------------------------# +# File: .gitlab/ci/ccscs-jobs.yml +# Author: Kelly Thompson +# Date: Monday, Jun 01, 2020, 15:00 pm +# Purpose: gitlab suid runner commands for LANL ccs-net machines. This script will be run to test +# pull requests. +# Note: Copyright (C) 2020-2021 Triad National Security, LLC., All rights reserved. +#--------------------------------------------------------------------------------------------------# + +#--------------------------------------------------------------------------------------------------# +# Machine-wide default job settings +#--------------------------------------------------------------------------------------------------# + +.ccscs: + variables: + http_proxy: "http://proxyout.lanl.gov:8080" + https_proxy: "http://proxyout.lanl.gov:8080" + DRACO_SOURCE_DIR: ${CI_PROJECT_DIR} + DRACO_BINARY_DIR: ${CI_PROJECT_DIR}/build/j + DRACO_BINARY_DIR: ${CI_PROJECT_DIR}/build/d + DRACO_SOURCE_DIR: ${CI_PROJECT_DIR}/draco + DEPLOY: "FALSE" + +#--------------------------------------------------------------------------------------------------# +# Build Templates +#--------------------------------------------------------------------------------------------------# + +.ccscs_merge: + extends: .ccscs + variables: + CTEST_MODE: Experimental + script: + - /bin/bash -l -c ${CI_PROJECT_DIR}/.gitlab/ci/gitlab-ci-run-tests.sh + only: + - merge_requests + +.ccscs_style_checks: + extends: .ccscs + variables: + CTEST_MODE: Experimental + script: + - /bin/bash -l -c ${CI_PROJECT_DIR}/.gitlab/ci/gitlab-ci-style-checks.sh + only: + - merge_requests + +.ccscs_nightly: + extends: .ccscs + variables: + CTEST_MODE: Nightly + CTEST_BUILD_NAME: ${DRACO_ENV}-${CMAKE_BUILD_TYPE} + script: + - /bin/bash -l -c ${CI_PROJECT_DIR}/.gitlab/ci/gitlab-nightly-regress.sh + only: + - schedules + +#--------------------------------------------------------------------------------------------------# +# Merge Requests +#--------------------------------------------------------------------------------------------------# + +# FAST TEST + +ccs1_style_cxx: + tags: + - ccscs1-setuid-shared + stage: fasttest + extends: .ccscs_style_checks + variables: + CMAKE_BUILD_TYPE: Release + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + SITE_ID: ccscs1 + +ccs3_deb_gcc_1020: + tags: + - ccscs3-setuid-shared + stage: fasttest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + CTEST_NPROC: 32 + SITE_ID: ccscs3 + +ccs2_c4_scalar: + tags: + - ccscs2-setuid-shared + stage: fasttest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + EXTRA_CMAKE_ARGS: -DDRACO_C4=SCALAR + CTEST_NPROC: 56 + SITE_ID: ccscs2 + +# ---------- regular TEST (only starts if all FASTTEST pass) ---------- # + +ccs3_deb_llvm11: + tags: + - ccscs3-setuid-shared + stage: test + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: llvm11 + ARCH: x86_64 + CTEST_NPROC: 32 + SITE_ID: ccscs3 + +ccs2_autodoc: + tags: + - ccscs2-setuid-shared + stage: test + extends: .ccscs_merge + allow_failure: true + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + AUTODOC: "ON" + SITE_ID: ccscs2 + + +# ---------- SLOW TEST (only starts if all regular TEST pass) ---------- # + +ccs3_rel_llvm11: + tags: + - ccscs3-setuid-shared + stage: slowtest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Release + DRACO_ENV: llvm11 + ARCH: x86_64 + CTEST_NPROC: 32 + SITE_ID: ccscs3 + +ccs3_rel_gcc_1020: + tags: + - ccscs3-setuid-shared + stage: slowtest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Release + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + CTEST_NPROC: 32 + SITE_ID: ccscs3 + +ccs3_fulldiag: + tags: + - ccscs3-setuid-shared + stage: slowtest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + # DRACO_TIMING=1 enables caliper timers. + # DRACO_DIAGNOSTICS=7 enables FPE traps. + EXTRA_CMAKE_ARGS: -DDRACO_DIAGNOSTICS=7 -DDRACO_TIMING=1 -DDRACO_LIBRARY_TYPE=STATIC + CTEST_NPROC: 32 + SITE_ID: ccscs3 + +ccs2_gcov: + tags: + - ccscs2-setuid-shared + stage: slowtest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + # --> Overall coverage rate: + # --> lines......: 91.2% (34056 of 37338 lines) + # --> functions..: 56.7% (8676 of 15304 functions) + # ... + # --> Total:|91.2% 37338|56.7% 15k| + EXTRA_CMAKE_ARGS: -DCODE_COVERAGE=ON + CTEST_NPROC: 56 + SITE_ID: ccscs2 + +ccs2_valgrind: + tags: + - ccscs2-setuid-shared + stage: slowtest + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + EXTRA_CMAKE_ARGS: -DENABLE_MEMORYCHECK=ON -DCTEST_MEMORYCHECK_SUPPRESSIONS_FILE=/scratch/regress/ccsradregress/valgrind_suppress.txt + EXTRA_CTEST_ARGS: -L memcheck -LE nomemcheck + CTEST_NPROC: 20 + SITE_ID: ccscs2 + +# ---------- Optional ---------- # + +ccs3_clangtidy: + tags: + - ccscs3-setuid-shared + # stage: optional + stage: fasttest + # allow_failure: true + extends: .ccscs_merge + variables: + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: llvm11 + ARCH: x86_64 + CTEST_NPROC: 32 + EXTRA_CMAKE_ARGS: -DDRACO_STATIC_ANALYZER=clang-tidy + SITE_ID: ccscs3 + +#--------------------------------------------------------------------------------------------------# +# Nightly Regression +#--------------------------------------------------------------------------------------------------# + +rel_gcc_10.2.0_regress: + tags: + - ccscs2-setuid-shared + extends: .ccscs_nightly + variables: + SITE_ID: ccscs2 + CMAKE_BUILD_TYPE: Release + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + # EXTRA_CMAKE_ARGS: -DDRACO_DIAGNOSTICS=7 -DDRACO_TIMING=1 -DDRACO_LIBRARY_TYPE=STATIC + BUILD_FLAGS: -j 20 + CTEST_NPROC: 32 + # TEST_EXCLUSIONS: tracking_tstU3 + +deb_gcc_10.2.0_regress: + tags: + - ccscs2-setuid-shared + extends: .ccscs_nightly + variables: + SITE_ID: ccscs2 + CMAKE_BUILD_TYPE: Debug + DRACO_ENV: gcc1020-ompi4 + ARCH: x86_64 + # EXTRA_CMAKE_ARGS: -DDRACO_DIAGNOSTICS=7 -DDRACO_TIMING=1 -DDRACO_LIBRARY_TYPE=STATIC + BUILD_FLAGS: -j 20 + CTEST_NPROC: 32 + # TEST_EXCLUSIONS: tracking_tstU3 + AUTODOCDIR: /ccs/codes/radtran/autodoc + +#--------------------------------------------------------------------------------------------------# +# end .gitlab/ci/ccscs-jobs.yml +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/check_style.sh b/.gitlab/ci/check_style.sh new file mode 100755 index 000000000..04ff95668 --- /dev/null +++ b/.gitlab/ci/check_style.sh @@ -0,0 +1,448 @@ +#!/bin/bash +# -*- Mode: sh -*- + +# ------------------------------------------------------------------------------------------------ # +# File : .gitlab/ci/check_style.sh +# Date : Thursday, May 20, 2021, 16:31 pm +# Author: Kelly Thompson +# Note : Copyright (C) 2021, Triad National Security, LLC., All rights are reserved. +# +# Runs various lint programs in the current directory and list locally modified files that are not +# compliant with the current coding standard (see .clang_format in the top level source directory.) +# - clang-format for C and C++ code. +# - emacs for Fortran90 +# - cmake-format and cmake-tidy for CMake scripts. +# ------------------------------------------------------------------------------------------------ # + +#--------------------------------------------------------------------------------------------------# +# Environment +#--------------------------------------------------------------------------------------------------# + +# Enable job control +set -m + +# protect temp files +umask 0077 + +# load some common bash functions +rscriptdir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" || exit ) +if ! [[ -d $rscriptdir ]]; then + rscriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +fi +if [[ -f "$rscriptdir/common.sh" ]]; then + # shellcheck source=.gitlab/ci/common.sh + source "$rscriptdir/common.sh" +else + echo " " + echo "FATAL ERROR: Unable to locate Draco's bash functions: " + echo " looking for .../.gitlab/ci//common.sh" + echo " searched rscriptdir = $rscriptdir" + exit 1 +fi +export rscriptdir + +#--------------------------------------------------------------------------------------------------# +# Support functions +#--------------------------------------------------------------------------------------------------# +print_use() +{ + echo " " + echo "Usage: ${0##*/} -f -t" + echo " " + echo "All arguments are optional." + echo " -f Show diff and fix files (when possible)." + echo -n " -t Run as a pre-commit check, print list of non-conformant files and return with" + echo " exit code = 1 (implies -d)." + echo " " +} + +#--------------------------------------------------------------------------------------------------# +# Sanity Checks +#--------------------------------------------------------------------------------------------------# + +# clang-format must be in the PATH +if [[ ${CLANG_FORMAT_VER} ]]; then + cfver="-${CLANG_FORMAT_VER}" +else + cfver="" +fi +# Assume applications have version postfix. +gcf=$(which "git-clang-format${cfver}") +cf=$(which "clang-format${cfver}") +# if not found, try to find applications w/o version postfix. +if ! [[ -f "${gcf}" ]]; then + gcf=$(which git-clang-format) +fi +if ! [[ -f "${cf}" ]]; then + gcf=$(which clang-format) +fi +# if still not found, abort. +if [[ ! "${gcf}" ]]; then + echo "ERROR: git-clang-format${cfver} was not found in your PATH." + echo "pwd=" + pwd + echo "which git-clang-format${cfver}" + echo "$gcf" + exit 1 +#else +# echo "Using $gcf --binary $cf" +fi +if [[ ! "${cf}" ]]; then + echo "ERROR: clang-format${cfver} was not found in your PATH." + echo "pwd=" + pwd + echo "which clang-format${cfver}" + echo "$cf" + echo "which git" + which git + exit 1 +fi + +#--------------------------------------------------------------------------------------------------# +# Default values +#--------------------------------------------------------------------------------------------------# +fix_mode=0 # show diffs AND modify code +foundissues=0 # 0 == ok + +#--------------------------------------------------------------------------------------------------# +# Command options +#--------------------------------------------------------------------------------------------------# + +while getopts ":fht" opt; do +case $opt in +f) fix_mode=1 ;; # also modify code (as possible) +h) print_use; exit 0 ;; +t) fix_mode=0 ;; +\?) echo "" ;echo "invalid option: -$OPTARG"; print_use; exit 1 ;; +:) echo "" ;echo "option -$OPTARG requires an argument."; print_use; exit 1 ;; +esac +done + +# common ancestor of feature branch (MR) and develop +echo -ne "\n--------------------------------------------------------------------------------\n" +currentbranch=$(git rev-parse --abbrev-ref HEAD) +comancest=$(git merge-base origin/develop "${currentbranch}") +echo "==> current branch : ${currentbranch}" +echo -ne "\n==> common ancestor hash : ${comancest}\n\n" + +# ------------------------------------------------------------------------------------------------ # +# List of modified files +# - Used for cmake-format and Emacs F90 processing. +# ------------------------------------------------------------------------------------------------ # + +modifiedfiles=$(git diff "${comancest}" --name-only) +# all files +modifiedfiles=$(echo "${modifiedfiles}" | sort -u) +echo "Files modified in this MR:" +for file in $modifiedfiles; do + echo " - $file" +done + +#--------------------------------------------------------------------------------------------------# +# Test C++ code with git-clang-format +#--------------------------------------------------------------------------------------------------# + +echo -ne "\n--------------------------------------------------------------------------------\n" +echo -ne "Checking modified C/C++ code for style conformance...\n\n" + +patchfile_c=$(mktemp /tmp/gcf.patch.XXXXXXXX) + +# don't actually modify the files (originally we compared to branch 'develop', but let's try +# ORIG_HEAD or maybe use CI variables like TRAVIS_BRANCH or CI_MERGE_REQUEST_TARGET_BRANCH_NAME). +# run "git branch -a" +target_branch=$comancest +echo "Looking at code changes compared to target branch = $target_branch" +cmd="${gcf} --binary ${cf} -f --diff --extensions hh,cc,cu $target_branch" +run "${cmd}" &> "${patchfile_c}" + +# if the patch file has the string "no modified files to format", the check passes. +if [[ $(grep -c "no modified files" "${patchfile_c}") != 0 ]] || \ + [[ $(grep -c "clang-format did not modify any files" "${patchfile_c}") != 0 ]]; then + echo "PASS: Changes to C++ sources conform to this project's style requirements." +else + foundissues=1 + echo "FAIL: some C++ files do not conform to this project's style requirements:" + # Modify files, if requested + if [[ ${fix_mode} == 1 ]]; then + echo -e " The following patch has been applied to your file.\n" + run "git apply $patchfile_c" + cat "${patchfile_c}" + else + echo -ne " run clang-formt -i < file> to fix files, or" + echo -e " run ${0##*/} with option -f to automatically apply this patch.\n" + cat "${patchfile_c}" + fi +fi +rm -f "${patchfile_c}" + +#--------------------------------------------------------------------------------------------------# +# Test CMake script for formatting issues +#--------------------------------------------------------------------------------------------------# + +CMF=$(which cmake-format) +if [[ $CMF ]]; then + CMFVER=$("$CMF" --version) + if [[ $(version_gt "0.5.0" "${CMFVER}") ]]; then + echo "WARNING: Your version of cmake-format is too old. Expecting v 0.5+. Pre-commit-hook" + echo " partially disabled (cmake-format, cmake-lint)" + unset CMF + fi + if ! [[ -f "${rscriptdir}/../../.cmake-format.py" ]]; then + unset CMF; + echo "WARNING: Missing .cmake-format.py configuration file, skipping cmake-format checks." + fi +fi + +DIFFVER=$(diff --version | head -n 1 | sed -e 's/.*[ ]\([0-9.]\)/\1/') +[[ $(version_gt "3.4" "${DIFFVER}") ]] && DIFFCOLOR="--color" + +if [[ -x "${CMF}" ]]; then + + echo -ne "\n--------------------------------------------------------------------------------\n" + echo -ne "Checking modified CMake code for style conformance...\n\n" + + patchfile_cmf=$(mktemp /tmp/cmf.patch.XXXXXXXX) + + # file types to parse. + FILE_EXTS=".cmake" + FILE_ENDINGS_INCLUDE="CMakeLists.txt" + # FILE_ENDINGS_EXLCUDE=".cmake.in" + + # Loop over all modified cmake files. Create one patch containing all changes to these files + for file in $modifiedfiles; do + + # ignore file if we do check for file extensions and the file does not match any of the + # extensions specified in $FILE_EXTS + if ! matches_extension "$file"; then continue; fi + + file_nameonly=$(basename "${file}") + tmpfile1="/tmp/cmf-${file_nameonly}" + echo "==> cmake-format ${file}" + cp -f "${file}" "${tmpfile1}" + $CMF -c "${rscriptdir}/../../.cmake-format.py" -i "${tmpfile1}" &> /dev/null + # color output is possible if diff -version >= 3.4 with option `--color` + diff ${DIFFCOLOR} -u "${file}" "${tmpfile1}" | \ + sed -e "1s|--- |--- a/|" -e "2s|+++ ${tmpfile1}|+++ b/${file}|" >> "$patchfile_cmf" + rm "${tmpfile1}" + + done + + unset FILE_EXTS + unset FILE_ENDINGS_INCLUDE + unset FILE_ENDINGS_EXCLUDE + + # If the patch file is size 0, then no changes are needed. + if [[ -s "$patchfile_cmf" ]]; then + foundissues=1 + echo "FAIL: some CMake files do not conform to this project's style requirements:" + # Modify files, if requested + if [[ ${fix_mode} == 1 ]]; then + echo -e " The following patch has been applied to your file.\n" + run "git apply $patchfile_cmf" + cat "${patchfile_cmf}" + else + echo -e " run cmake-format -i to fix files, or" + echo -e " run ${0##*/} with option -f to automatically apply this patch.\n" + cat "${patchfile_cmf}" + fi + else + echo "PASS: Changes to CMake sources conform to this project's style requirements." + fi + rm -f "${patchfile_cmf}" + +else + echo -ne "\n==> Skipping cmake-format checks\n\n" +fi + +#--------------------------------------------------------------------------------------------------# +# Test CMake script for lint issues +#--------------------------------------------------------------------------------------------------# + +CML=$(which cmake-lint) +if [[ $CML ]]; then + CMLVER=$("$CML" --version) + if [[ $(version_gt "0.5.0" "${CMLVER}") ]]; then + echo "WARNING: Your version of cmake-lint is too old. Expecting v 0.5+. Pre-commit-hook" + echo " partially disabled (cmake-format, cmake-lint)" + unset CML + fi + if ! [[ -f "${rscriptdir}/../../.cmake-format.py" ]]; then unset CML; fi +fi + +if [[ -x $CML ]]; then + + echo -ne "\n--------------------------------------------------------------------------------\n" + echo -ne "Checking modified CMake code for lint conformance...\n\n" + + # file types to parse. + FILE_EXTS=".cmake" + FILE_ENDINGS_INCLUDE="CMakeLists.txt" + # FILE_ENDINGS_EXLCUDE=".cmake.in" + export FILE_EXTS FILE_ENDINGS_INCLUDE + + cml_issues=0 + # Loop over all modified cmake files. Create one patch containing all changes to these files + for file in $modifiedfiles; do + + # ignore file if we do check for file extensions and the file does not match any of the + # extensions specified in $FILE_EXTS + if ! matches_extension "$file"; then continue; fi + + printf "==> cmake-lint %s\n" "$file" + $CML --suppress-decoration "${file}" && echo -ne "==> cmake-lint ${file} ... OK\n" || cml_issues=1 + + done + + unset FILE_EXTS + unset FILE_ENDINGS_INCLUDE + unset FILE_ENDINGS_EXCLUDE + + # If the patch file is size 0, then no changes are needed. + if [[ ${cml_issues} -gt 0 ]]; then + foundissues=1 + echo -e "FAIL: some CMake files do not conform to this project's style requirements:" + echo -e " You must fix these issues manuall. Run cmake-lint to view issues again." + else + echo "PASS: Changes to CMake sources conform to this project's style requirements." + fi + +else + echo -ne "\n==> Skipping cmake-lint checks\n\n" +fi + +#--------------------------------------------------------------------------------------------------# +# Check mode (Test F90 code indentation with emacs and bash) +#--------------------------------------------------------------------------------------------------# + +# Defaults ---------------------------------------- +EMACS=$(which emacs) +if [[ $EMACS ]]; then + EMACSVER=$("$EMACS" --version | head -n 1 | sed -e 's/.*Emacs //') + if [[ $(version_gt "24.0.0" "${EMACSVER}") ]]; then + echo "WARNING: Your version of emacs is too old. Expecting v 24.0+. Pre-commit-hook partially" + echo " disabled (f90 indentation)" + unset EMACS + fi +fi + +if [[ -x "$EMACS" ]]; then + + echo -ne "\n--------------------------------------------------------------------------------\n" + echo -e "Checking modified F90 code for style conformance (indentation)...\n" + + patchfile_f90=$(mktemp /tmp/emf90.patch.XXXXXXXX) + + # file types to parse. + FILE_EXTS=".f90 .F90 .f .F" + # FILE_ENDINGS_INCLUDE="CMakeLists.txt" + # FILE_ENDINGS_EXLCUDE=".cmake.in" + export FILE_EXTS + + # Loop over all modified F90 files. Create one patch containing all changes to these files + for file in $modifiedfiles; do + + # ignore file if we do check for file extensions and the file does not match any of the + # extensions specified in $FILE_EXTS + if ! matches_extension "$file"; then continue; fi + + file_nameonly=$(basename "${file}") + tmpfile1="/tmp/f90-format-${file_nameonly}" + cp -f "${file}" "${tmpfile1}" + "$EMACS" -batch "${tmpfile1}" -l "${rscriptdir}/../environment/git/f90-format.el" \ + -f emacs-format-f90-sources &> /dev/null + # color output is possible if diff -version >= 3.4 with option `--color` + diff -u "${file}" "${tmpfile1}" | \ + sed -e "1s|--- |--- a/|" -e "2s|+++ ${tmpfile1}|+++ b/${file}|" >> "$patchfile_f90" + rm "${tmpfile1}" + + done + + # If the patch file is size 0, then no changes are needed. + if [[ -s "$patchfile_f90" ]]; then + foundissues=1 + echo -n "FAIL: some F90 files do not conform to this project's style requirements:" + # Modify files, if requested + if [[ "${fix_mode}" == 1 ]]; then + echo -e " The following patch has been applied to your file.\n" + run "git apply $patchfile_f90" + cat "$patchfile_f90" + else + echo -ne " run ${0##*/} with option -f to automatically apply this patch.\n" + cat "$patchfile_f90" + fi + else + echo -n "PASS: Changes to F90 sources conform to this project's style requirements." + fi + rm -f "${patchfile_f90}" + +fi + +#--------------------------------------------------------------------------------------------------# +# Check mode (Test F90 code line length with bash) +#--------------------------------------------------------------------------------------------------# + +echo -ne "\n--------------------------------------------------------------------------------\n" +echo -e "Checking modified F90 code for style conformance (line length)...\n" + +tmpfile2=$(mktemp /tmp/f90-format-line-len.XXXXXXXX) + +# Loop over all modified F90 files. Create one patch containing all changes to these files +for file in $modifiedfiles; do + + # ignore file if we do check for file extensions and the file does not match any of the extensions + # specified in $FILE_EXTS + if ! matches_extension "$file"; then continue; fi + + header_printed=0 + lineno=0 + + # shellcheck disable=SC2162 + while read line; do + (( lineno++ )) + # Exceptions: + # - Long URLs + exception=$(echo "${line}" | grep -i -c http) + if [[ ${#line} -gt 100 && ${exception} == 0 ]]; then + if [[ ${header_printed} == 0 ]]; then + { + echo -e "\nFile: ${file} [code line too long]\n"; + echo " line length content"; + echo -n " ------ ------ -------------------------------------------------------------"; + echo "-------------------" + } >> "${tmpfile2}" + header_printed=1 + fi + printf " %-6s %-6s %s\n" "${lineno}" "${#line}" "${line}" >> "${tmpfile2}" + fi + # reset exception flag + exception=0 + done < "${file}" +done + +unset FILE_EXTS +unset FILE_ENDINGS_INCLUDE +unset FILE_ENDINGS_EXCLUDE + +# If there are issues, report them +if [[ $(wc -l < "${tmpfile2}") -gt 0 ]]; then + foundissues=1 + echo -ne "FAIL: some F90 files do not conform to this project's style requirements:\n" + cat "$tmpfile2" + echo -ne "\nPlease reformat lines listed above to fit into 80 columns and attempt running\n" + echo -ne "${0##*/} again. These issues cannot be fixed with the -f option." +fi + +#--------------------------------------------------------------------------------------------------# +# Done +#--------------------------------------------------------------------------------------------------# + +echo -ne "\n--------------------------------------------------------------------------------\n" +echo "Done" +echo -ne "\n--------------------------------------------------------------------------------\n" + +# Return code: 0==ok,1==bad +exit $foundissues + +#--------------------------------------------------------------------------------------------------# +# End check_style.sh +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/common.sh b/.gitlab/ci/common.sh new file mode 100755 index 000000000..55d4fed22 --- /dev/null +++ b/.gitlab/ci/common.sh @@ -0,0 +1,89 @@ +#!/bin/bash -l +# -*- Mode: sh -*- +#--------------------------------------------------------------------------------------------------# +# File : .gitlab/ci/common.sh +# Date : Tuesday, May 31, 2016, 14:48 pm +# Author: Kelly Thompson +# Note : Copyright (C) 2016-2021, Triad National Security, LLC., All rights are reserved. +#--------------------------------------------------------------------------------------------------# +# +# Summary: Misc bash functions useful during development of code. + +shopt -s extglob # Allow variable as case condition + +#--------------------------------------------------------------------------------------------------# +# Helpful functions +#--------------------------------------------------------------------------------------------------# + +function dracohelp() +{ +echo -e "Bash functions defined by Draco:\n\n" +echo -e "\nUse 'type ' to print the full content of any function.\n" +echo -e "\ndie - Print an error message and exit." +echo -e "\nrun - echo a command and then evaluate it." +echo -e "\nfn_exists - return true if the named bash function is defined." +echo -e "\nversion_gt - compare versions" +echo -e "\n" +} + +#--------------------------------------------------------------------------------------------------# +# Function Definitions +#--------------------------------------------------------------------------------------------------# + +# Print an error message and exit. e.g.: cd $dir || die "can't change dir to $dir". +function die () { echo " "; echo "FATAL ERROR: $1"; exit 1;} + +# Echo a command and then run it. +function run () +{ + echo "==> $1"; if [[ "${dry_run:-no}" == "no" ]]; then eval "$1"; fi +} + +# Return 0 if provided name is a bash function. +function fn_exists () +{ + type "$1" 2>/dev/null | grep -c 'is a function' +} + +#--------------------------------------------------------------------------------------------------# +# Compare versions +# +# Example: +# +# EMACSVER=`emacs --version | head -n 1 | sed -e 's/.*Emacs //'` +# if `version_gt "24.0.0" $EMACSVER` ; then echo yes; fi +#--------------------------------------------------------------------------------------------------# +function version_gt() +{ + test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; +} + +#--------------------------------------------------------------------------------------------------# +# check whether the given file matches any of the set extensions +# +# Example: +# +# FILE_EXTS=".f90 .F90" +# FILE_ENDINGS_INCLUDE="CMakeLists.txt" +# FILE_ENDINGS_EXCLUDE="_f.h _f77.h _f90.h" +# for file in $modified_files; do +# if ! matches_extension $file; then continue; fi +# +# done +#--------------------------------------------------------------------------------------------------# +matches_extension() { + local filename + local extension + local end + local ext + filename=$(basename "$1") + extension=".${filename##*.}" + for end in $FILE_ENDINGS_EXCLUDE; do [[ "$filename" == *"$end" ]] && return 1; done + for end in $FILE_ENDINGS_INCLUDE; do [[ "$filename" == *"$end" ]] && return 0; done + for ext in $FILE_EXTS; do [[ "$ext" == "$extension" ]] && return 0; done + return 1 +} + +#--------------------------------------------------------------------------------------------------# +# End common.sh +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/environments.sh b/.gitlab/ci/environments.sh new file mode 100755 index 000000000..863892763 --- /dev/null +++ b/.gitlab/ci/environments.sh @@ -0,0 +1,58 @@ +#!/bin/bash -l +# -*- Mode: sh -*- +#--------------------------------------------------------------------------------------------------# +# File : .gitlab/ci/environments.sh +# Date : Monday, Jun 01, 2020, 15:43 pm +# Author: Kelly Thompson +# Note : Copyright (C) 2020-2021, Triad National Security, LLC., All rights are reserved. +#--------------------------------------------------------------------------------------------------# + +echo "==> Setting up CI environment..." +echo " SITE_ID = ${SITE_ID}" +[[ -n "${SCHEDULER_PARAMETERS}" ]] && echo " Using: salloc ${SCHEDULER_PARAMETERS}" + +case ${SITE_ID} in + darwin | ccscs*) ;; + *) die ".gitlab/ci/environments.sh :: SITE_ID not recognized, SITE_ID = ${SITE_ID}" ;; +esac + +echo " DRACO_ENV = ${DRACO_ENV}" +echo " ARCH = ${ARCH}" + +if [[ "${SITE_ID}" == "darwin" ]]; then + DRACO_ARCH=$(/usr/projects/draco/vendors/bin/target_arch) + run "module use --append /projects/draco/Modules" + case ${DRACO_ENV} in + arm-gcc930 | power9-gcc930-smpi | power9-xl16117 | x64-gcc930 | x64-intel1905 ) ;; + *) die ".gitlab/ci/environments.sh :: DRACO_ENV not recognized, DRACO_ENV = ${DRACO_ENV}" ;; + esac + if [[ "${MPIARCH:-notset}" == "openmpi" ]]; then + disable_openib=$(sinfo -N -n "${HOSTNAME}" -o %all | grep -c ib:none) + if [[ ${disable_openib} != 0 ]]; then + export MPI_PREFLAGS="--mca btl ^openib" + fi + fi + export DRACO_ARCH + +elif [[ "${SITE_ID}" =~ "ccscs" ]]; then + run "module use --append /ccs/codes/radtran/Modules" + export PATH=/scratch/vendors/bin:$PATH # clang-format + case ${DRACO_ENV} in + gcc1020* | llvm11 ) ;; + *) die ".gitlab/ci/environments.sh :: DRACO_ENV not recognized, DRACO_ENV = ${DRACO_ENV}" ;; + esac +fi + +#--------------------------------------------------------------------------------------------------# +# Setup requested CI environment +#--------------------------------------------------------------------------------------------------# + +run "module load draco/${DRACO_ENV}" +if [[ "${SLURM_JOB_PARTITION}" =~ "volta" || "${SLURM_JOB_PARTITION}" =~ "gpu" ]]; then + module load cuda/11.0 +fi +run "module list" + +#--------------------------------------------------------------------------------------------------# +# End environments.sh +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/gitlab-ci-run-tests.sh b/.gitlab/ci/gitlab-ci-run-tests.sh new file mode 100755 index 000000000..225c354c3 --- /dev/null +++ b/.gitlab/ci/gitlab-ci-run-tests.sh @@ -0,0 +1,149 @@ +#!/bin/bash -l +#--------------------------------------------------------------------------------------------------# +# File : ./.gitlab/ci/gitlab-ci-run-tests.sh +# Date : Tuesday, Jun 02, 2020, 12:28 pm +# Author: Kelly Thompson +# Note : Copyright (C) 2020-2021, Triad National Security, LLC., All rights reserved. +#--------------------------------------------------------------------------------------------------# + +# preliminaries and environment +set -e +# shellcheck source=.gitlab/ci/common.sh +source "${JAYENNE_SOURCE_DIR}/.gitlab/ci/common.sh" +# shellcheck source=.gitlab/ci/environments.sh +source "${JAYENNE_SOURCE_DIR}/.gitlab/ci/environments.sh" + +if [[ $(which lscpu | grep -c lscpu) -gt 0 ]]; then + # run "lscpu" + # NPROC=`lscpu | grep CPU\(s\) | head -n 1 | awk '{ print $2 }'` + NPROC=$(grep -c processor < /proc/cpuinfo) +fi + +[[ -z "${CTEST_NPROC}" ]] && CTEST_NPROC=$NPROC || echo "limiting CTEST_NPROC = $CTEST_NPROC" +[[ -z "${MAXLOAD}" ]] && MAXLOAD=$NPROC || echo "limiting MAXLOAD = $MAXLOAD" + +#if [[ -d /ccs/opt/texlive/2018/texmf-dist/tex/latex/newunicodechar ]]; then +# TEXINPUTS="${TEXINPUTS:+${TEXINPUTS}:}/ccs/opt/texlive/2018/texmf-dist/tex/latex/newunicodechar" +#fi + +# Report number of cores/threads, etc. +# run "lstopo --no-io --no-bridges --no-icaches" + +echo -e "\n========== printenv ==========\n" +[[ -z "${SLURM_NODELIST}" ]] || echo "SLURM_NODELIST = ${SLURM_NODELIST}" +echo "HOSTNAME = ${HOSTNAME}" +echo -e "NPROC = ${NPROC}\n" +echo -e "CTEST_NPROC = ${CTEST_NPROC}\n" +echo -e "MAXLOAD = ${MAXLOAD}\n" +echo -e "EXTRA_CMAKE_ARGS = ${EXTRA_CMAKE_ARGS}\n" +echo -e "CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}\n" +echo -e "EXTRA_CTEST_ARGS = ${EXTRA_CTEST_ARGS}\n" +if [[ "${AUTODOC}" == "ON" ]]; then + run "which doxygen" + run "which latex" + run "which pdflatex" + echo "TEXINPUTS = ${TEXINPUTS}" +fi +if [[ "${EXTRA_CTEST_ARGS}" =~ memcheck ]]; then + run "which valgrind" +fi +printenv >& environment.log +run "pwd" + +[[ "${EXTRA_CMAKE_ARGS}" =~ "CODE_COVERAGE" ]] && CODECOV=ON || CODECOV=OFF + +#--------------------------------------------------------------------------------------------------# +# Setup compiler flags +#--------------------------------------------------------------------------------------------------# + +for i in C CXX Fortran CUDA; do + + # Turn build warnings into fatal errors + if [[ "${DRACO_ENV}" =~ "xl16" ]]; then + eval export ${i}_FLAGS+=\" -qhalt=w\" + else + if [[ "${i}" == "Fortran" ]] && [[ "${FC}" =~ "ifort" ]]; then + # do not add the flag + echo "Skip adding -Werror to ${i}_FLAGS (FC = ${FC})" + elif [[ "${i}" == "CUDA" ]]; then + eval export ${i}_FLAGS+=\" -Werror all-warnings\" + else + # Works for gcc, llvm, and icpc (not ifort). + eval export ${i}_FLAGS+=\" -Werror\" + fi + fi + + # Enable coverage for Debug builds + if [[ ${CMAKE_BUILD_TYPE} == Debug ]] && [[ "${CODECOV}" == "ON" ]] ; then + eval export ${i}_FLAGS+=\" --coverage\" + fi +done +echo "C_FLAGS = $C_FLAGS" +echo "CXX_FLAGS = $CXX_FLAGS" +# shellcheck disable=2154 +echo "Fortran_FLAGS = $Fortran_FLAGS" +echo "CUDA_FLAGS = $CUDA_FLAGS" + +# Tweak MPI settings +# export OMPI_MCA_btl=self,sm +export OMPI_MCA_btl=^openib + +#--------------------------------------------------------------------------------------------------# +# Draco (build and install libraries only, no tests) +#--------------------------------------------------------------------------------------------------# + +echo -e "\n========== Draco ==========\n" +if ! [[ -d "${DRACO_BINARY_DIR}" ]]; then run "mkdir -p ${DRACO_BINARY_DIR}"; fi +if ! [[ -d "${DRACO_SOURCE_DIR}" ]]; then + run "cd ${CI_PROJECT_DIR} && git clone https://github.com/lanl/Draco draco" +else + run "cd ${DRACO_SOURCE_DIR} && git pull" +fi +run "cd ${DRACO_BINARY_DIR}" +echo -e "\n========" +run "cmake ${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DBUILD_TESTING=OFF ${DRACO_SOURCE_DIR}" +echo " " +run "make -j -l $MAXLOAD" +# don't echo all of the install commands to the terminal. +# shellcheck disable=2086 +echo "make -j -l $MAXLOAD install" +make -j -l "$MAXLOAD" install &> /dev/null + +#--------------------------------------------------------------------------------------------------# +# Jayenne +#--------------------------------------------------------------------------------------------------# + +echo -e "\n========== Jayenne ==========\n" +if ! [[ -d ${JAYENNE_BINARY_DIR} ]]; then run "mkdir -p ${JAYENNE_BINARY_DIR}"; fi +run "cd ${JAYENNE_BINARY_DIR}" +echo -e "\n========" +run "cmake ${EXTRA_CMAKE_ARGS} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} ${JAYENNE_SOURCE_DIR}" +echo " " +#if [[ -f CMakeCache.txt ]]; then +# echo -e "\n===== CMakeCache.txt =====\n" +# run "cat CMakeCache.txt" +#fi +echo -e "\n========\n" +if [[ "${AUTODOC}" == "ON" ]]; then + run "make -j -l $MAXLOAD autodoc" +else + run "make -j -l $MAXLOAD" # VERBOSE=1 + echo -e "\n========\n" + run "ctest ${EXTRA_CTEST_ARGS} -j ${CTEST_NPROC} --test-load ${MAXLOAD} --output-on-failure --stop-on-failure" +fi + +# Generate a coverage report (text and html) +if [[ "${CODECOV}" == "ON" ]]; then + run "make covrep" +fi + +if [[ ${CTEST_MODE} == "Nightly" ]]; then + echo -e "\n========\n" + echo "Nighlty regressions should run a ctest in script mode and post to CDash" +fi + +echo -e "\n======== end .gitlab-ci-run-tests.sh ==========\n" + +#--------------------------------------------------------------------------------------------------# +# End .gitlab/ci/gitlab-ci-run-tests.sh +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/gitlab-ci-style-checks.sh b/.gitlab/ci/gitlab-ci-style-checks.sh new file mode 100755 index 000000000..0b05c4d53 --- /dev/null +++ b/.gitlab/ci/gitlab-ci-style-checks.sh @@ -0,0 +1,40 @@ +#!/bin/bash -l +#--------------------------------------------------------------------------------------------------# +# File : ./.gitlab/ci/gitlab-style-checks.sh +# Date : Tuesday, Jun 02, 2020, 12:28 pm +# Author: Kelly Thompson +# Note : Copyright (C) 2020, Triad National Security, LLC., All rights are reserved. +#--------------------------------------------------------------------------------------------------# + +# preliminaries and environment +set -e +#shellcheck source=.gitlab/ci/common.sh +source "${JAYENNE_SOURCE_DIR}/.gitlab/ci/common.sh" +#shellcheck source=.gitlab/ci/environments.sh +source "${JAYENNE_SOURCE_DIR}/.gitlab/ci/environments.sh" + +echo -e "\n========== printenv ==========\n" +if ! [[ "${SLURM_NODELINST:-notset}" == "notset" ]]; then + echo "SLURM_NODELIST = ${SLURM_NODELIST}" +fi +echo "HOSTNAME = ${HOSTNAME}" +# printenv +# run "module avail" +printenv | grep "CI_" +echo " " + +#--------------------------------------------------------------------------------------------------# +# Style check +#--------------------------------------------------------------------------------------------------# + +run "cd $JAYENNE_SOURCE_DIR" +if [[ $(git branch | grep -c develop) == 0 ]]; then + run "git branch develop origin/develop" +fi +run "${JAYENNE_SOURCE_DIR}/.gitlab/ci/check_style.sh" -t + +echo -e "\n======== end .gitlab-ci-run-tests.sh ==========\n" + +#--------------------------------------------------------------------------------------------------# +# End .gitlab/ci/gitlab-ci-run-tests.sh +#--------------------------------------------------------------------------------------------------# diff --git a/.gitlab/ci/superlinter.yml b/.gitlab/ci/superlinter.yml new file mode 100644 index 000000000..52964540a --- /dev/null +++ b/.gitlab/ci/superlinter.yml @@ -0,0 +1,50 @@ +#--------------------------------------------------------------------------------------------------# +# File: .gitlab/ci/superlinter.yml +# Author: Kelly Thompson +# Date: Wednesday, Jan 13, 2021, 20:57 pm +# Purpose: gitlab suid runner commands for superlinter (https://github.com/github/super-linter). +# Note: Copyright (C) 2020-2021 Triad National Security, LLC., All rights reserved. +# +# See Also: https://gitlab.com/snippets/1988376 +#--------------------------------------------------------------------------------------------------# + +superlinter: + stage: fasttest + image: github/super-linter:latest + script: [ "true" ] + only: + - merge_requests +# allow_failure: true + variables: + RUN_LOCAL: "true" + DEFAULT_WORKSPACE: $CI_PROJECT_DIR + ANSIBLE_DIRECTORY: $CI_PROJECT_PATH + LINTER_RULES_PATH: $CI_PROJECT_PATH/.github/linters + DEFAULT_BRANCH: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME + ERROR_ON_MISSING_EXEC_BIT: "true" + FILTER_REGEX_EXCLUDE: (.*[.]cc$|/test/) + PYTHON_FLAKE8_CONFIG_FILE: .flake8 + SUPPRESS_POSSUM: "true" + LOG_LEVEL: "VERBOSE" + VALIDATE_ALL_CODEBASE: "false" + VALIDATE_BASH: "true" + VALIDATE_BASH_EXEC: "true" + VALIDATE_DOCKERFILE: "true" + VALIDATE_ENV: "true" + VALIDATE_HTML: "true" + VALIDATE_JSON: "true" + VALIDATE_LATEX: "true" + VALIDATE_LUA: "true" + VALIDATE_MARKDOWN: "true" + VALIDATE_PERL: "true" + VALIDATE_PYTHON: "true" + VALIDATE_PYTHON_FLAKE8: "true" + VALIDATE_POWERSHELL: "true" + VALIDATE_RUBY: "true" + VALIDATE_SHELL_SHFMT: "true" + VALIDATE_XML: "true" + VALIDATE_YAML: "true" + +#--------------------------------------------------------------------------------------------------# +# end .gitlab/ci/superlinter.yml +#--------------------------------------------------------------------------------------------------#