From 0f6dfd538b8791a8ea908db430ba739672098313 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Tue, 16 Aug 2016 16:57:38 -0600 Subject: [PATCH 01/20] Upgrade history tools to python Full change list: 1) Remove old shell-based tools and update calls to use python versions 2) Move most functionality in old shell tools into hist_utils.py 3) Make thin python wrapper programs to access hist_utils from the command line 4) Do st_archive as LAST step in run_indv so that coupler_log_path is not needed 5) Fix ERR test 6) Update fake tests to create a fake hist file 7) Large refactor of bless_test_results --- scripts/Testing/Testcases/NOC_script | 15 +- scripts/Tools/baseline_gen_comp | 265 ----------- scripts/Tools/component_compare.sh | 323 -------------- scripts/Tools/component_compare_baseline | 55 +++ scripts/Tools/component_compare_move | 53 +++ scripts/Tools/component_compare_move.sh | 185 -------- scripts/Tools/component_compare_test | 58 +++ scripts/Tools/component_compare_test.sh | 411 ------------------ scripts/Tools/component_compgen_baseline.sh | 350 --------------- scripts/Tools/component_generate.sh | 238 ---------- scripts/Tools/component_generate_baseline | 55 +++ utils/python/CIME/SystemTests/eri.py | 7 +- utils/python/CIME/SystemTests/err.py | 2 +- utils/python/CIME/SystemTests/ssp.py | 4 +- .../CIME/SystemTests/system_tests_common.py | 127 ++---- utils/python/CIME/bless_test_results.py | 112 ++--- utils/python/CIME/hist_utils.py | 172 ++++++++ .../python/tests/scripts_regression_tests.py | 3 +- 18 files changed, 484 insertions(+), 1951 deletions(-) delete mode 100755 scripts/Tools/baseline_gen_comp delete mode 100755 scripts/Tools/component_compare.sh create mode 100755 scripts/Tools/component_compare_baseline create mode 100755 scripts/Tools/component_compare_move delete mode 100755 scripts/Tools/component_compare_move.sh create mode 100755 scripts/Tools/component_compare_test delete mode 100755 scripts/Tools/component_compare_test.sh delete mode 100755 scripts/Tools/component_compgen_baseline.sh delete mode 100755 scripts/Tools/component_generate.sh create mode 100755 scripts/Tools/component_generate_baseline create mode 100644 utils/python/CIME/hist_utils.py diff --git a/scripts/Testing/Testcases/NOC_script b/scripts/Testing/Testcases/NOC_script index df03281e181..f7978cbbf1d 100755 --- a/scripts/Testing/Testcases/NOC_script +++ b/scripts/Testing/Testcases/NOC_script @@ -75,10 +75,10 @@ else endif echo "moving relevant history files to suffix with command " >>& $TESTSTATUS_LOG -echo "$SCRIPTSROOT/Tools/component_compare_move.sh -rundir $RUNDIR -testcase $CASE -suffix base" >>& $TESTSTATUS_LOG +echo "$SCRIPTSROOT/Tools/component_compare_move base" >>& $TESTSTATUS_LOG echo "" >>& $TESTSTATUS_LOG -$SCRIPTSROOT/Tools/component_compare_move.sh -rundir $RUNDIR -testcase $CASE -suffix "base" +$SCRIPTSROOT/Tools/component_compare_move base #====================================================================== # do an initial run test with NINST 2, with the @@ -117,10 +117,10 @@ else endif echo "moving relevant history files to suffix with commands " >>& $TESTSTATUS_LOG -echo "$SCRIPTSROOT/Tools/component_compare_move.sh -rundir $RUNDIR -testcase $CASE -suffix inst2mod" >>& $TESTSTATUS_LOG +echo "$SCRIPTSROOT/Tools/component_compare_move inst2mod" >>& $TESTSTATUS_LOG echo "" >>& $TESTSTATUS_LOG -$SCRIPTSROOT/Tools/component_compare_move.sh -rundir $RUNDIR -testcase $CASE -suffix "inst2mod" +$SCRIPTSROOT/Tools/component_compare_move inst2mod #====================================================================== # Check test status for all relevant component history files @@ -130,10 +130,7 @@ echo "DONE ${CASEBASEID} : (test finished, successful coupler log) " >>& $TESTST echo "" >>& $TESTSTATUS_LOG echo "obtaining test functionality from history files with commands " >>& $TESTSTATUS_LOG -echo "$SCRIPTSROOT/Tools/component_compare_test.sh -rundir $RUNDIR -testcase $CASE -testcase_base $CASEBASEID -suffix1 base -suffix2 multiinst -msg compare .base with .inst2mod" >>& $TESTSTATUS_LOG +echo "$SCRIPTSROOT/Tools/component_compare_test base multiinst" >>& $TESTSTATUS_LOG echo "" >>& $TESTSTATUS_LOG -$SCRIPTSROOT/Tools/component_compare_test.sh -rundir $RUNDIR -testcase $CASE -testcase_base $CASEBASEID -suffix1 "base" -suffix2 "multiinst" -msg "compare .base with .inst2mod" >>& $TESTSTATUS_OUT - - - +$SCRIPTSROOT/Tools/component_compare_test base multiinst >>& $TESTSTATUS_OUT diff --git a/scripts/Tools/baseline_gen_comp b/scripts/Tools/baseline_gen_comp deleted file mode 100755 index c6cfd4c3d60..00000000000 --- a/scripts/Tools/baseline_gen_comp +++ /dev/null @@ -1,265 +0,0 @@ -#!/bin/bash - -# Bill Sacks -# 4-15-2015 - -#====================================================================== -# Overview -#====================================================================== - -# This script generates baselines and/or compares with previous baselines, for -# all tests that were run in a single test suite. It then outputs the test -# status (PASS/FAIL/etc.) for each comparison / generation. -# -# This is meant to be run after all tests in a given test suite are -# complete. Some typical uses are: (1) re-comparing against baselines if some -# baselines did not yet exist when the test suite was run, (2) comparing against -# a different set of baselines from the ones originally specified when you -# created the test suite, (3) generating baselines after the fact (if you ran -# the test suite without -generate). -# -# In addition to some of the options given to create_test_suite, you also need -# to provide the script with (1) the unique test ID of the test suite, and (2) -# the directory in which the various test run directories can be found. -# -# cprnc should be in your path, but if it isn't, the script tries to -# find cprnc in its yellowstone location -# -# See usage message for details on inputs. - -#====================================================================== -# Limitations -#====================================================================== - -# - Doesn't look at the PASS/FAIL status of a test. This means, for -# example, that a -generate may copy files to the baseline directory -# from a test that failed; these files might not be the correct, -# end-of-run history files, and so could later result in a -# misleading FAIL message from a -compare. -# -# - Only looks for history files in the run directory. This will fail -# to find a history file if things have been moved to the short-term -# archive directory. This could be fixed to look in the short-term -# archive directory if a file can't be found in the run directory, -# as is done in testcase_end. I haven't done this yet, partly -# because it adds an extra directory that the user has to specify -# (because we don't have access to the DOUT_S_ROOT variable), and -# partly because this shouldn't matter for most tests. - -#====================================================================== -# Testing -#====================================================================== - -# There currently is no unit test script for this. Here are the tests -# that should be done on this script: -# -# - Missing arguments -# -# - generate only -# -# - compare only -# -# - compare and generate - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo " Generates baselines and/or compares with previous baselines," - echo " for all tests that were run in a single test suite." - echo " It then outputs the test status (PASS/FAIL/etc.) for each comparison / generation." - echo "" - echo " This is meant to be run after all tests in a given test suite are complete." - echo " Some typical uses are:" - echo " (1) re-comparing against baselines if some baselines did not yet exist" - echo " when the test suite was run" - echo " (2) comparing against a different set of baselines from the ones originally" - echo " specified when you created the test suite" - echo " (3) generating baselines after the fact (if you ran the test suite without -generate)" - echo "" - echo " The -baselineroot, -generate and -compare arguments will usually match the arguments" - echo " given to create_test_suite. The -testid argument will match the -testid argument" - echo " given to create_test_suite, if provided; otherwise, it will be the testid created" - echo " on the fly by that script." - echo "" - echo " cprnc should be in your path, but if it isn't, the script tries to find cprnc" - echo " in its yellowstone location" - echo "" - echo "OPTIONS" - echo " -baselineroot Path to directory containing baselines (required)" - echo "" - echo " -generate Tag name to use for generation (optional)" - echo " A directory with this name is created in baselineroot," - echo " if it doesn't already exist." - echo " Either -generate or -compare must be given" - echo "" - echo " -compare Tag name to use for comparison (optional)" - echo " Baselines are found in a directory with the name of this tag" - echo " within baselineroot." - echo " Either -generate or -compare must be given" - echo "" - echo " -testid ID of the test suite on which to run this script (required)" - echo " (e.g., 123456)" - echo "" - echo " -runloc Path to directory containing test run directories (optional)" - echo " A given test's run directory can be found in:" - echo " \$runloc/\$CASE/run" - echo " If not given, defaults to: $runloc_default" - echo "" - echo " -help Print this help message and exit" - echo "" - echo "EXAMPLES" - echo " $progname" - echo " -baselineroot /glade/scratch/\$USER/cesm_baselines" - echo " -generate cesm1_1_beta17" - echo " -compare cesm1_1_beta16" - echo " -testid 123456" - echo " -runloc /glade/scratch/\$USER" - echo " This will find directories in /glade/scratch/\$USER whose name ends with" - echo " the testid, 123456. For each such test (say, SMS.f09_g16.IG.yellowstone_intel.GC.123456)" - echo " it will find the last history files in the run directory of that test," - echo " for each possible component (cpl and clm - including h0, h1, etc.)." - echo " It will then copy those history files to" - echo " /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta17/SMS.f09_g16.IG.yellowstone_intel" - echo " and will compare that history file with the existing baseline in" - echo " /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/SMS.f09_g16.IG.yellowstone_intel" - echo "" -} - -#====================================================================== -# Set parameters -#====================================================================== -# If cprnc isn't in your path, this is where to find it -cprnc_default=/glade/p/cesm/cseg/tools/cprnc/cprnc - -#====================================================================== -# Begin main script -#====================================================================== - -progname=`basename $0` -tools_dir=`dirname $0` - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -baselineroot='' -generate_tag='' -compare_tag='' -testid='' - -runloc_default="/glade/scratch/$USER" -runloc=$runloc_default - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -baselineroot ) - baselineroot=$2 - shift - ;; - -generate ) - generate_tag=$2 - shift - ;; - -compare ) - compare_tag=$2 - shift - ;; - -testid ) - testid=$2 - shift - ;; - -runloc ) - runloc=$2 - shift - ;; - -help ) - Usage - exit 0 - ;; - * ) - echo "Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 - ;; - esac - shift -done - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet -if [ -z "$baselineroot" ]; then - echo "$progname: baselineroot must be provided" >&2 - error=1 -fi -# Either -generate or -compare must be provided, but not necessarily both -if [[ -z "$generate_tag" && -z "$compare_tag" ]]; then - echo "$progname: At least one of generate_tag or compare_tag must be provided" >&2 - error=1 -fi -if [ -z "$testid" ]; then - echo "$progname: testid must be provided" >&2 - error=1 -fi -if [ -z "$runloc" ]; then - echo "$progname: runloc must be provided" >&2 - error=1 -fi - -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 -fi - -#---------------------------------------------------------------------- -# Determine path to cprnc -#---------------------------------------------------------------------- -cprnc_path=`command -v cprnc` -if [ $? -gt 0 ]; then - # cprnc not found in path; use default - cprnc_path=$cprnc_default -fi - -#---------------------------------------------------------------------- -# Loop over directories with the given testid -#---------------------------------------------------------------------- -tests=`cd $runloc; ls -1d *.${testid}` -for testcase in $tests; do - # testcase will look like: SMS.T31_g37.IG4804.yellowstone_intel.134426 - # optionally with .C or .G or .GC before the testid - # Form testcase_base by stripping off the trailing testid and the optional .C, .G or .GC: - testcase_base=`echo $testcase | perl -p -e "s/(\.G?C?)?\.\Q$testid\E\$//"` - - if [ "$testcase_base" == "sharedlibroot" ]; then - continue - fi - - rundir=${runloc}/${testcase}/run - - # We need to call component_compgen_baseline.sh separately for compare & - # generate, because currently it can only do one at a time (because only one - # baseline_dir can be provided, and the baseline_dir is assumed to contain - # the actual tag, rather than being a directory with all tags' baselines) - - if [ -n "$compare_tag" ]; then - echo "--- Baseline Comparison ---: " - ${tools_dir}/component_compgen_baseline.sh -baseline_dir ${baselineroot}/${compare_tag}/${testcase_base} -test_dir ${rundir} -testcase ${testcase} -testcase_base ${testcase_base} -compare_tag ${compare_tag} -cprnc_exe ${cprnc_path} - fi - - if [ -n "$generate_tag" ]; then - echo "--- Baseline Generation ---: " - ${tools_dir}/component_compgen_baseline.sh -baseline_dir ${baselineroot}/${generate_tag}/${testcase_base} -test_dir ${rundir} -testcase ${testcase} -testcase_base ${testcase_base} -generate_tag ${generate_tag} -cprnc_exe ${cprnc_path} - fi - - echo "" -done - diff --git a/scripts/Tools/component_compare.sh b/scripts/Tools/component_compare.sh deleted file mode 100755 index 97d4b809de6..00000000000 --- a/scripts/Tools/component_compare.sh +++ /dev/null @@ -1,323 +0,0 @@ -#!/bin/bash - -# Bill Sacks -# 8-23-12 - -#====================================================================== -# Overview -#====================================================================== - -# This script does the '-compare' step for a single test case and a -# single history file. -# -# It is intended to be used by another script, such as -# component_gen_comp. -# -# The definition of the result is similar to the compare_hist status -# in testcase_end, but may differ slightly. Other than possible minor -# differences, though, one use of this script can be: You start -# baseline runs and test runs at the same time (both using -# 'create_test_suite ... -clean off'); test runs finish before -# baseline generation finished (leading to BFAILs); then you can run -# this script with model=cpl to do the comparisons after the baseline -# generation completes (but first you will need to copy the cpl.hi.nc -# files in each baseline directory to cpl.h.nc). -# -# cprnc should be in your path, but if it isn't, the script tries to -# find cprnc in its yellowstone location -# -# See usage message for details on inputs & return value. - -#====================================================================== -# Testing -#====================================================================== - -# There currently is no unit test script for this. Here are the unit -# tests that should be done on this script: -# -# - Missing arguments -# - return status should be "UNDEF" -# -# - test_hist='', no baseline -# - return status should be "BFAIL_NA" -# - example: tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist DOES_NOT_EXIST -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist ''` -# -# - Given test history file doesn't exist, no baseline -# - return status should be "BFAIL_NA" -# - example: tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist DOES_NOT_EXIST -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist DOES_NOT_EXIST` -# -# - test_hist='' (but baseline file exists) -# - return status should be "FAIL" -# - example: tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist ''` -# -# - Given test history file doesn't exist (but baseline file exists) -# - return status should be "FAIL" -# - example: tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist DOES_NOT_EXIST` -# -# - Given test history file exists, but no baseline -# - return status should be "BFAIL" -# - example: component_compare.sh -baseline_dir $baseline_dir -baseline_hist DOES_NOT_EXIST -test_dir $test_dir -test_hist ERS_Ld211_P192x1.f19_g16.I_2000_CNDV_CROP.yellowstone_intel_crop.G.061555.clm2.h0.1998-07-30-00000.nc -# -# - History comparison fails: difference in values (e.g., set up by -# manually modifying values in a file) -# - return status should be "FAIL" -# - example (after running the PASS example from component_generate.sh, then running: ncap2 -s 'vvel[$time,$level,$y0,$x0]=1.7' ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029.cism.h.2046-01-01-00000.nc test.nc): tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist test.nc` -# -# - History comparison fails: different time stamp on file (e.g., from -# a different case from the one used to generate the baseline) -# (this is really testing the behavior of cprnc more than this script -# per se) -# - return status should be "FAIL" -# - example (after running the PASS example from component_generate.sh): tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/SMS_D.f09_g16.TG.bluefire_ibm.C.114029/run -test_hist SMS_D.f09_g16.TG.bluefire_ibm.C.114029.cism.h.0011-01-01-00000.nc` -# -# - History comparison passes -# - return status should be "PASS" -# - example (after running the PASS example from component_generate.sh): tst=`component_compare.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029.cism.h.2046-01-01-00000.nc` - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo "RETURN VALUE" - echo " String of the format 'STATUS:Extra info about failure'" - echo " The string before the colon is the test status." - echo " The string after the colon is extra information about a failure" - echo " (this may be blank, but the colon will still be present)" - echo "" - echo " Possible values for STATUS are:" - echo " UNDEF : undefined result; this includes incorrect usage" - echo " BFAIL_NA: no baseline file and no history file for the test case" - echo " (note that BFAIL_NA is only a problem if it occurs for all possible" - echo " history extensions for a given component)" - echo " BFAIL : no baseline file, but there IS a history file for the test case" - echo " FAIL : comparison fails (including: baseline exists but no history file for the test case)" - echo " PASS : success" - echo "" - echo "OPTIONS" - echo " -cprnc_exe Full pathname to cprnc executable " - echo " -baseline_dir Full path to the baseline directory for this test (required)" - echo " -baseline_hist Name used for history file in the baseline directory (required)" - echo " -test_dir Full path to the directory containing history files for this test (required)" - echo " NOTE: You must have write permission in this directory (cprnc.out file is put there)" - echo " -test_hist Name of history file in the test directory (required)" - echo " (it is NOT an error for this to be an empty string, but this may generate a FAIL;" - echo " this will be the case when there are no history files for a component in the test directory)" - echo " -help Print this help message and exit" - echo "" - echo "EXAMPLES" - echo " TEST=SMS.T31_g37.IG4804.bluefire_ibm" - echo " CASE=\$TEST.GC.134426" - echo "" - echo " $progname -baseline_dir /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST" - echo " -baseline_hist clm.h.nc" - echo " -test_dir /glade/scratch/\$USER/\$CASE/run" - echo " -test_hist \$CASE.clm.h0.0001-12.nc" - echo " This will compare /glade/scratch/\$USER/\$CASE/run/\$CASE.clm.h0.0001-12.nc" - echo " with /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST/clm.h.nc" - echo "" - echo " $progname -baseline_dir /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST" - echo " -baseline_hist clm.h.nc" - echo " -test_dir /glade/scratch/\$USER/\$CASE/run" - echo " -test_hist ''" - echo " This will generate a FAIL because there is no history file to compare with the baseline" - echo " (or a BFAIL_NA if /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST/clm.h.nc also doesn't exist)" - echo "" -} - -# Given a relative path, convert to an absolute path -# (from http://www.lancejian.com/2011/04/13/get-absolute-path-of-the-running-bash-script.html) -function absolute_path { - relative_path="$1" - absolute_path=`cd "$relative_path"; pwd` - echo "$absolute_path" -} - -# Prints the test status and an info string, separated by ':' -# Inputs: -# status: test status -# info: optional auxiliary info about test failure -function print_result { - status="$1" - info="$2" - - echo "${status}:COMPCOMPARE${info}" -} - -#====================================================================== -# Begin main script -#====================================================================== - -progname=`basename $0` - -# need absolute path (rather than relative path) because we use this -# path after we have cd'ed to another location -tools_dir=$(absolute_path `dirname $0`) - -#---------------------------------------------------------------------- -# Set default return values -#---------------------------------------------------------------------- -status='UNDEF' -info='' - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -cprnc_exe='' -baseline_dir='' -baseline_hist='' -test_dir='' - -# test_hist will often be the empty string, so we initialize it to -# something else so we can check whether it has been provided -test_hist_init='XXX_NOTGIVEN_XXX' -test_hist=$test_hist_init - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -cprnc_exe ) - cprnc_exe=$2 - shift - ;; - -baseline_dir ) - baseline_dir=$2 - shift - ;; - -baseline_hist ) - baseline_hist=$2 - shift - ;; - -test_dir ) - test_dir=$2 - shift - ;; - -test_hist ) - test_hist=$2 - shift - ;; - -help ) - Usage - exit 0 - ;; - * ) - echo "$progname: Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - # return default values for status & info - print_result $status "$info" - exit 1 - ;; - esac - shift -done - - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet - -if [ -z "$cprnc_exe" ]; then - echo "$progname: cprnc_exe must be provided" >&2 - error=1 -fi -if [ -z "$baseline_dir" ]; then - echo "$progname: baseline_dir must be provided" >&2 - error=1 -fi -if [ -z "$baseline_hist" ]; then - echo "$progname: baseline_hist must be provided" >&2 - error=1 -fi -if [ -z "$test_dir" ]; then - echo "$progname: test_dir must be provided" >&2 - error=1 -fi -if [ "$test_hist" = "$test_hist_init" ]; then - echo "$progname: test_hist must be provided" >&2 - error=1 -fi - -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - # return default values for status & info - print_result $status "$info" - exit 1 -fi - -#---------------------------------------------------------------------- -# Determine whether baseline file exists -#---------------------------------------------------------------------- -if [ -f $baseline_dir/$baseline_hist ]; then - baseline_exists=1 -else - baseline_exists=0 -fi - -#---------------------------------------------------------------------- -# Determine whether test history file exists -#---------------------------------------------------------------------- -# Test history file may be the empty string if there is known to be no -# history file for this component. Another possibility is that the -# given test history file doesn't exist; this is more likely to be an -# error in the usage of the script rather than a real test failure, so -# we may want to rethink the status value in that case. -if [ -z "$test_hist" ]; then - test_exists=0 -elif [ -f $test_dir/$test_hist ]; then - test_exists=1 -else - test_exists=0 -fi - -#---------------------------------------------------------------------- -# If baseline and/or test history file don't exist, return appropriate -# error status -#---------------------------------------------------------------------- - -if [[ $baseline_exists -eq 0 && $test_exists -eq 0 ]]; then - status="BFAIL_NA" - info="neither baseline nor test history file exists" - print_result $status "$info" - exit 2 -elif [[ $baseline_exists -eq 0 && $test_exists -eq 1 ]]; then - status="BFAIL" - info="baseline history file does not exist" - print_result $status "$info" - exit 2 -elif [[ $baseline_exists -eq 1 && $test_exists -eq 0 ]]; then - status="FAIL" - info="no history file in test case" - print_result $status "$info" - exit 2 -fi - -# Note: at this point, we know that there is both a baseline history -# file and a test history file - -#---------------------------------------------------------------------- -# Compare history files, get test status -# Put output in a file named ${test_dir}/${test_hist}.cprnc.out -#---------------------------------------------------------------------- - -# We cd to test_dir so that $testhist.out.out is put there -# (note that this assumes that the user has write permission in test_dir) - -curdir=`pwd` -cd $test_dir - -cprnc_output=$($cprnc_exe $test_dir/$test_hist $baseline_dir/$baseline_hist | tee ${test_hist}.cprnc.out 2> /dev/null ) - -if [[ $cprnc_output == *IDENTICAL* ]]; then - print_result PASS "$info" - exit 0 -else - print_result FAIL "$info" - exit 3 -fi diff --git a/scripts/Tools/component_compare_baseline b/scripts/Tools/component_compare_baseline new file mode 100755 index 00000000000..f6e25bd0d62 --- /dev/null +++ b/scripts/Tools/component_compare_baseline @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +""" +Compares current component history files against baselines +""" + +from standard_script_setup import * + +from CIME.case import Case +from CIME.hist_utils import compare_baseline + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s [] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Compare baselines \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + CIME.utils.setup_standard_logging_options(parser) + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory") + + parser.add_argument("-b", "--baseline-dir", + help="Use custom baseline dir") + + args = parser.parse_args(args[1:]) + + CIME.utils.handle_standard_logging_options(args) + + return args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + caseroot, baseline_dir = parse_command_line(sys.argv, description) + with Case(caseroot) as case: + success, comments = compare_baseline(case, baseline_dir) + print comments + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/scripts/Tools/component_compare_move b/scripts/Tools/component_compare_move new file mode 100755 index 00000000000..7823e13f06e --- /dev/null +++ b/scripts/Tools/component_compare_move @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +""" +Change the suffix for the most recent batch of hist files. This allows us +to save these results if we want to run the case again. +""" + +from standard_script_setup import * + +from CIME.case import Case +from CIME.hist_utils import move + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s suffix [] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Setup case \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + CIME.utils.setup_standard_logging_options(parser) + + parser.add_argument("suffix", + help="Suffix to append to hist files") + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory") + + args = parser.parse_args(args[1:]) + + CIME.utils.handle_standard_logging_options(args) + + return args.suffix, args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + suffix, caseroot = parse_command_line(sys.argv, description) + with Case(caseroot) as case: + move(case, suffix) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/scripts/Tools/component_compare_move.sh b/scripts/Tools/component_compare_move.sh deleted file mode 100755 index 693b913e661..00000000000 --- a/scripts/Tools/component_compare_move.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/bash - -# Mariana Vertenstein -# 11-07-2014 - - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo " Moves component history file to new name with suffix in the testcase directory" - echo " If there are multiple history file types for the component (e.g., h0 & h1)," - echo " it uses one of each type (e.g., the latest h0 file and the latest h1 file)." - echo "" - echo "OPTIONS" - echo " -rundir Path to directory containing test run directories (optional)" - echo " A given test's run directory can be found in: \$rundir/\$CASE/run" - echo "" - echo " -testcase case name (identical to $CASE)" - echo "" - echo " -suffix Suffix name to attache to testcase" - echo "" - echo " -help Print this help message and exit" - echo "" - echo "EXAMPLES" - echo " $progname" - echo " -rundir /glade/scratch/\$USER -testcase ERS.f19_g16.FC5.yellowstone_intel -suffix init" - echo "" -} - -#====================================================================== -# Begin main script -#====================================================================== - -tools_dir=`dirname $0` - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -testcase='' -rundir='' -suffix='' -add_iop='' - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -rundir ) - rundir=$2 - shift - ;; - -testcase ) - testcase=$2 - shift - ;; - -suffix ) - suffix=$2 - shift - ;; - -add_iop ) - add_iop=$2 - shift - ;; - -help ) - Usage - exit 0 - ;; - * ) - echo "Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 - ;; - esac - shift -done - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet -if [ -z "$rundir" ]; then - echo "$progname: rundir must be provided" >&2 - error=1 -fi -if [ -z "$testcase" ]; then - echo "$progname: testcase must be provided" >&2 - error=1 -fi -if [ -z "$suffix" ]; then - echo "$progname: suffix must be provided" >&2 - error=1 -fi -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 -fi - -#------------------------------------------------------------------ -# Loop over models -#------------------------------------------------------------------ - -if [ -n "$add_iop" ]; then - echo " add_iop is set" - suffix="${suffix}_${add_iop}" -fi - -cd $rundir -models=(cam cice clm2 pop cism cpl rtm mosart) -for model in ${models[*]}; do - - if [ "$model" = "cism" ]; then - extensions=(h) - elif [ "$model" = "clm2" ]; then - extensions=(h0 h1 h2 h3 h4 h5) - elif [ "$model" = "cpl" ]; then - extensions=(hi) - elif [ "$model" = "cice" ]; then - extensions=(h) - elif [ "$model" = "cam" ]; then - extensions=(h0 h1 h2 h3 h4 h5 h6 h7) - elif [ "$model" = "pop" ]; then - extensions=(h) - elif [ "$model" = "rtm" ]; then - extensions=(h0 h1 h2) - elif [ "$model" = "mosart" ]; then - extensions=(h0 h1 h2) - fi - - #------------------------------------------------------------------ - # Loop over history file extensions - #------------------------------------------------------------------ - - for extension in ${extensions[*]}; do - - #-------------------------------------------------------------- - # Find last component hist file in this run directory, and - # determine corresponding name of the baseline file (used for - # either generation or comparison) - #-------------------------------------------------------------- - # Note that we find the last alphabetically rather than by time - # stamp, because the last by time stamp can be non-deterministic. - # Note that we need a * after ${model} to capture multi-instance - # output - - if [ "$suffix" = "multiinst" ]; then - instances=(_0001 _0002) - else - instances=(none _0001 _0002) - fi - for inst in ${instances[*]}; do - if [ "$inst" = "none" ]; then - test_hist=`ls -1 ${testcase}.${model}.${extension}.*.nc 2>/dev/null | tail -1` - else - test_hist=`ls -1 ${testcase}.${model}${inst}.${extension}.*.nc 2>/dev/null | tail -1` - fi - - if [[ -f ${test_hist} ]]; then - test_hist_suffix=${test_hist}.${suffix} - cd "$rundir"; - - # remove suffix file if already present - if [[ -f ${test_hist_suffix} ]]; then - rm "$test_hist_suffix" - fi - - # FIXME(bja, 2014-11) temp change to test if cp fixes the - # problem with mfilt in restart tests. If it does, then - # rename this script to component_compare_copy.sh ! - /bin/cp "$test_hist" "$test_hist_suffix" - fi - done - - done - -done # loop over history file extensions - - - diff --git a/scripts/Tools/component_compare_test b/scripts/Tools/component_compare_test new file mode 100755 index 00000000000..aae11911e49 --- /dev/null +++ b/scripts/Tools/component_compare_test @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +""" +Compares two component history files in the testcase directory +""" + +from standard_script_setup import * + +from CIME.case import Case +from CIME.hist_utils import compare_test + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s suffix1 suffix2 [] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Setup case \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + CIME.utils.setup_standard_logging_options(parser) + + parser.add_argument("suffix1", + help="The suffix of the first set of files") + + parser.add_argument("suffix2", + help="The suffix of the second set of files") + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory") + + args = parser.parse_args(args[1:]) + + CIME.utils.handle_standard_logging_options(args) + + return args.suffix1, args.suffix2, args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + suffix1, suffix2, caseroot = parse_command_line(sys.argv, description) + with Case(caseroot) as case: + success, comments = compare_test(case, suffix1, suffix2) + print comments + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/scripts/Tools/component_compare_test.sh b/scripts/Tools/component_compare_test.sh deleted file mode 100755 index 1412fc3a49b..00000000000 --- a/scripts/Tools/component_compare_test.sh +++ /dev/null @@ -1,411 +0,0 @@ -#!/bin/bash - -# Mariana Vertenstein -# 11-07-2014 - - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo " Compares two component history files in the testcase directory" - echo " If there are multiple history file types for the component (e.g., h0 & h1)," - echo " it uses one of each type (e.g., the latest h0 file and the latest h1 file)." - echo " It then outputs the test status (PASS/FAIL/etc.) for each comparison / generation." - echo "" - echo "OPTIONS" - echo " -rundir Path to directory containing test run directories (optional)" - echo " A given test's run directory can be found in: \$rundir/\$CASE/run" - echo "" - echo " -testcase Full name of testcase including testid" - echo "" - echo " -testcase_base Full name of testcase without testid" - echo "" - echo " -suffix1 Suffix to attach to first file to compare" - echo "" - echo " -suffix2 Suffix to attach to second file to compare" - echo "" - echo " -help Print this help message and exit" - echo "" -} - -# Given a status_and_info string, return just the status portion -# Inputs: -# - status_and_info: colon-delimited string of the format -# "STATUS:Extra info about failure". The string before the colon -# is the test status. The string after the colon is extra -# information abaout a failure (this may be blank, but the colon -# must be present). (It is okay for the extra info to itself -# contain one or more colons.) -function get_status { - local status_and_info="$1" - echo $status_and_info | cut -d ':' -f 1 -} - -# Given a status_and_info string, return just the info portion -# Inputs: -# - status_and_info: colon-delimited string of the format -# "STATUS:Extra info about failure". The string before the colon -# is the test status. The string after the colon is extra -# information abaout a failure (this may be blank, but the colon -# must be present). (It is okay for the extra info to itself -# contain one or more colons.) -function get_info { - local status_and_info="$1" - echo $status_and_info | cut -d ':' -f 2- -} - -# Prints the status of the test in a standardized format for test results -# Inputs: -# - status -# - info -# - testcase: name of test case -function print_status { - local status="$1" - local info="$2" - local testcase="$3" - - # Enclose info in parentheses - if [ -n "$info" ]; then - info_str="($info)" - else - info_str="" - fi - - # Print formatted test result - printf '%-3s %s compare\n' "$status" "${testcase}" - - echo "COMMENT $info" -} - -function print_comment { - local status="$1" - local info="$2" - local testcase="$3" - - # Enclose info in parentheses - if [ -n "$info" ]; then - info_str="($info)" - else - info_str="" - fi - - echo "COMMENT for $testcase $info" -} - -#====================================================================== -# Begin main script -#====================================================================== - -tools_dir=`dirname $0` - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -rundir='' -testcase='' -testcase_base='' -suffix1='' -suffix2='' -add_iop='' -msg='' - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -msg ) - msg=$2 - shift - ;; - -rundir ) - rundir=$2 - shift - ;; - -testcase ) - testcase=$2 - shift - ;; - -testcase_base ) - testcase_base=$2 - shift - ;; - -suffix1 ) - suffix1=$2 - shift - ;; - -suffix2 ) - suffix2=$2 - shift - ;; - -add_iop ) - add_iop=$2 - shift - ;; - -help ) - Usage - exit 0 - ;; - * ) - echo "Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 - ;; - esac - shift -done - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet -if [ -z "$rundir" ]; then - echo "$progname: rundir must be provided" >&2 - error=1 -fi -if [ -z "$testcase" ]; then - echo "$progname: testcase must be provided" >&2 - error=1 -fi -if [ -z "$testcase_base" ]; then - echo "$progname: testcase_base must be provided" >&2 - error=1 -fi -if [ -z "$suffix1" ]; then - echo "$progname: suffix1 must be provided" >&2 - error=1 -fi -if [ -z "$suffix2" ]; then - echo "$progname: suffix2 must be provided" >&2 - error=1 -fi -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 -fi - -#------------------------------------------------------------------ -# Determine location of cprnc -#------------------------------------------------------------------ -if [ -z $CASEROOT ]; then - echo " environment variable CASEROOT is not defined " - exit 1 -fi -cd $CASEROOT -cprnc_exe=`./xmlquery CCSM_CPRNC -value` - -#------------------------------------------------------------------ -# Loop over models -#------------------------------------------------------------------ - -cd $rundir -overall_status='PASS' - -models=(cam cice clm2 pop cism cpl rtm mosart) -for model in ${models[*]}; do - - if [ "$model" = "cism" ]; then - extensions=(h) - elif [ "$model" = "clm2" ]; then - extensions=(h0 h1 h2 h3 h4 h5) - elif [ "$model" = "cpl" ]; then - extensions=(hi) - elif [ "$model" = "cice" ]; then - extensions=(h) - elif [ "$model" = "cam" ]; then - extensions=(h0 h1 h2 h3 h4 h5 h6 h7 h8 h9 hs) - elif [ "$model" = "pop" ]; then - extensions=(h) - if [ "$suffix2" = "rest" ]; then - # Skip restart checks for pop! Temporary hack until MPAS goes in! - continue - fi - elif [ "$model" = "rtm" ]; then - extensions=(h0 h1 h2) - elif [ "$model" = "mosart" ]; then - extensions=(h0 h1 h2) - fi - - #------------------------------------------------------------------ - # Loop over history file extensions - #------------------------------------------------------------------ - for extension in ${extensions[*]}; do - - #-------------------------------------------------------------- - # Find last component hist files in this run directory, and - # determine corresponding name of the baseline file (used for - # either generation or comparison) - #-------------------------------------------------------------- - # Note that we find the last alphabetically rather than by time - # stamp, because the last by time stamp can be non-deterministic. - # Note that we need a * after ${model} to capture multi-instance - # output - #------------------------------------------------------------- - # Do comparison - #------------------------------------------------------------- - - if [[ "$testcase" =~ ^NC[KRO]_.* ]]; then - - hist1=`cd $rundir; ls -1 ${testcase}.${model}.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2_0001=`cd $rundir; ls -1 ${testcase}.${model}_0001.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - hist2_0002=`cd $rundir; ls -1 ${testcase}.${model}_0002.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - - if [[ -f ${hist1} ]] && [[ -f ${hist2_0001} ]] && [[ -f ${hist2_0002} ]] ; then - - if [ "$model" != "cpl" ]; then - # do all model comparisons except for cpl, since cpl history does not write out all instances - but just instance 1 - - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2_0001" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} and .${suffix2} for _0001)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2_0002" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} and .${suffix2} for _0002)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - - fi - fi - - elif [[ "$testcase" =~ .*_N2.* ]]; then - - hist1_0001=`cd $rundir; ls -1 ${testcase}.${model}_0001.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2_0001=`cd $rundir; ls -1 ${testcase}.${model}_0001.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - hist1_0002=`cd $rundir; ls -1 ${testcase}.${model}_0002.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2_0002=`cd $rundir; ls -1 ${testcase}.${model}_0002.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - - if [[ -f ${hist1_0001} ]] && [[ -f ${hist1_0002} ]] && [[ -f ${hist2_0001} ]] && [[ -f ${hist2_0002} ]] ; then - - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1_0001" -test_hist "$hist1_0002" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} for _0001 and .${suffix2} for _0001)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist2_0002" -test_hist "$hist2_0002" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} for _0002 and .${suffix2} for _0002)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - - fi - - - elif [[ "$testcase" =~ .*_IOP.* ]]; then - - if [ -z "$add_iop" ]; then - - # determine status of non-IOP test - - hist1=`cd $rundir; ls -1 ${testcase}.${model}.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2=`cd $rundir; ls -1 ${testcase}.${model}.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - - if [[ -f ${hist1} ]] && [[ -f ${hist2} ]] ; then - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} and .${suffix2} files)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - fi - fi - - if [ -n "$add_iop" ]; then - - # determine status of iop test - - hist1=`cd $rundir; ls -1 ${testcase}.${model}*.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2=`cd $rundir; ls -1 ${testcase}.${model}*.${extension}.*.nc.${suffix1}_${add_iop} 2>/dev/null | tail -1` - - if [[ -f ${hist1} ]] && [[ -f ${hist2} ]] ; then - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} and .${suffix1}_${add_iop} files)" "${testcase_base}" - fi - - hist1=`cd $rundir; ls -1 ${testcase}.${model}*.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - hist2=`cd $rundir; ls -1 ${testcase}.${model}*.${extension}.*.nc.${suffix2}_${add_iop} 2>/dev/null | tail -1` - - if [[ -f ${hist1} ]] && [[ -f ${hist2} ]] ; then - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix2} and .${suffix2}_${add_iop} files)" "${testcase_base}" - fi - - fi - - else - - # No _IOP_ or _N2_ attributes or multi-instance NCK_ or NCR_ tests - hist1=`cd $rundir; ls -1 ${testcase}.${model}.${extension}.*.nc.${suffix1} 2>/dev/null | tail -1` - hist2=`cd $rundir; ls -1 ${testcase}.${model}.${extension}.*.nc.${suffix2} 2>/dev/null | tail -1` - - if [[ -f ${hist1} ]] && [[ -f ${hist2} ]] ; then - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$rundir" -test_dir "$rundir" -baseline_hist "$hist1" -test_hist "$hist2" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - print_comment "$compare_status" "$compare_info: ${model}.${extension}.nc : test compare ${model}.${extension} (.${suffix1} and .${suffix2} files)" "${testcase_base}" - if [ "$compare_status" != "PASS" ]; then - overall_status="FAIL" - fi - elif [[ -f ${hist1} ]]; then - overall_status = "FAIL" - print_comment "file status" "File ${hist2} not found" "${testcase_base}" - elif [[ -f ${hist2} ]]; then - overall_status = "FAIL" - print_comment "file status" "File ${hist1} not found" "${testcase_base}" - fi - - fi - - done # loop over history file extensions - -done # loop over models - -if [ -z "$msg" ]; then - print_status "$overall_status" "$compare_info: test functionality summary" "${testcase_base}" -else - print_status "$overall_status" "$compare_info: test functionality summary ($msg)" "${testcase_base}" -fi - -if [[ $overall_status == FAIL ]]; then - exit 1 -else - exit 0 -fi - - - - - - - - - - diff --git a/scripts/Tools/component_compgen_baseline.sh b/scripts/Tools/component_compgen_baseline.sh deleted file mode 100755 index 6b5aa1cd393..00000000000 --- a/scripts/Tools/component_compgen_baseline.sh +++ /dev/null @@ -1,350 +0,0 @@ -#!/bin/bash - -# Mariana Vertenstein -# 11-07-2014 - - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo " Does a baseline comparison of history files or a baseline generation of history files for one test" - echo " If there are multiple history file types for the component (e.g., h0 & h1)," - echo " it uses one of each type (e.g., the latest h0 file and the latest h1 file)." - echo " It then outputs the test status (PASS/BFAIL/GFAIL/etc.) for each comparison / generation." - echo "" - echo " Exactly one of -generate_tag or -compare_tag should be provided." - echo " (They cannot both be provided, because only one -baseline_dir is supplied.)" - echo "" - echo "OPTIONS" - echo " -baseline_dir Path to directory containing baselines for this particular test (required)" - echo " If -compare_tag is given, this gives the path to the" - echo " baselines to compare against." - echo " If -generate_tag is given, this gives the path in which" - echo " the new baselines should be placed." - echo "" - echo " -test_dir Path to the given test's run directory (required)" - echo "" - echo " -testcase Full name of case including testid (required)" - echo "" - echo " -testcase_base Name of test case without testid, used for printing results (required)" - echo "" - echo " -generate_tag Tag to use for baseline generation (optional)" - echo "" - echo " -compare_tag Tag to use for baseline comparison (optional)" - echo "" - echo " -cprnc_exe Full pathname to cprnc executable (optional)" - echo " This is optional, but if not provided, then the environment" - echo " variable CASEROOT must be defined (in which case, CCSM_CPRNC" - echo " will be found from the xml files in CASEROOT)." - echo "" - echo " -help Print this help message and exit" - echo "" - echo "" -} - -# Given a status_and_info string, return just the status portion -# Inputs: -# - status_and_info: colon-delimited string of the format -# "STATUS:Extra info about failure". The string before the colon -# is the test status. The string after the colon is extra -# information abaout a failure (this may be blank, but the colon -# must be present). (It is okay for the extra info to itself -# contain one or more colons.) -function get_status { - local status_and_info="$1" - echo $status_and_info | cut -d ':' -f 1 -} - -# Given a status_and_info string, return just the info portion -# Inputs: -# - status_and_info: colon-delimited string of the format -# "STATUS:Extra info about failure". The string before the colon -# is the test status. The string after the colon is extra -# information abaout a failure (this may be blank, but the colon -# must be present). (It is okay for the extra info to itself -# contain one or more colons.) -function get_info { - local status_and_info="$1" - echo $status_and_info | cut -d ':' -f 2- -} - -# Prints the status of the test in a standardized format for test results -# Inputs: -# - status -# - info -# - testcase: name of test case -function print_status { - local status="$1" - local info="$2" - local testcase="$3" - local action="$4" - - # Enclose info in parentheses - if [ -n "$info" ]; then - info_str="($info)" - else - info_str="" - fi - - # Print formatted test result - printf '%-3s %s %s\n' "$status" "${testcase}" "${action}" - - echo "COMMENT $info" -} - -function print_comment { - local status="$1" - local info="$2" - local testcase="$3" - - # Enclose info in parentheses - if [ -n "$info" ]; then - info_str="($info)" - else - info_str="" - fi - - echo "COMMENT for $testcase $info" -} - -#====================================================================== -# Begin main script -#====================================================================== - -progname=`basename $0` -tools_dir=`dirname $0` - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -cprnc_exe='' -baseline_dir='' -test_dir='' -generate='' -compare_tag='' -msg='' - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -msg ) - msg=$2 - shift - ;; - -baseline_dir ) - baseline_dir=$2 - shift - ;; - -test_dir ) - test_dir=$2 - shift - ;; - -testcase ) - testcase=$2 - shift - ;; - -testcase_base ) - testcase_base=$2 - shift - ;; - -generate_tag ) - generate_tag=$2 - shift - ;; - -compare_tag ) - compare_tag=$2 - shift - ;; - -cprnc_exe ) - cprnc_exe=$2 - shift - ;; - - -help ) - Usage - exit 0 - ;; - * ) - echo "Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 - ;; - esac - shift -done - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet -if [ -z "$baseline_dir" ]; then - echo "$progname: baseline_dir must be provided" >&2 - error=1 -fi -if [ -z "$test_dir" ]; then - echo "$progname: test_dir must be provided" >&2 - error=1 -fi -if [ -z "$testcase" ]; then - echo "$progname: testcase must be provided" >&2 - error=1 -fi -if [ -z "$testcase_base" ]; then - echo "$progname: testcase_base must be provided" >&2 - error=1 -fi -if [ -z "$compare_tag" -a -z "$generate_tag" ]; then - echo "$progname: Must provide either -compare_tag or -generate_tag" >&2 - error=1 -fi -if [ -n "$compare_tag" -a -n "$generate_tag" ]; then - echo "$progname: Can only provide -compare_tag OR -generate_tag, not both" >&2 - echo "(this is because only one -baseline_dir is provided to this script)" >&2 - error=1 -fi - -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - exit 1 -fi - -#------------------------------------------------------------------ -# Determine location of cprnc, if not provided -#------------------------------------------------------------------ -if [ -z "$cprnc_exe" ]; then - if [ -z $CASEROOT ]; then - echo " environment variable CASEROOT is not defined " - exit 1 - fi - cd $CASEROOT - cprnc_exe=`./xmlquery CCSM_CPRNC -value` -fi - -#------------------------------------------------------------------ -# Loop over models -#------------------------------------------------------------------ - -overall_compare_status='PASS' -overall_generate_status='PASS' - -models=( cam cice cism clm2 cpl pop rtm mosart) -for model in ${models[*]}; do - if [ "$model" = "cism" ]; then - extensions=(h) - elif [ "$model" = "clm2" ]; then - extensions=(h0 h1 h2 h3 h4 h5) - elif [ "$model" = "cpl" ]; then - extensions=(hi) - elif [ "$model" = "cice" ]; then - extensions=(h) - elif [ "$model" = "cam" ]; then - extensions=(h0 h1 h2 h3 h4 h5 h6 h7) - elif [ "$model" = "pop" ]; then - extensions=(h) - elif [ "$model" = "rtm" ]; then - extensions=(h0 h1 h2) - elif [ "$model" = "mosart" ]; then - extensions=(h0 h1 h2) - fi - - #------------------------------------------------------------------ - # Loop over history file extensions - #------------------------------------------------------------------ - for extension in ${extensions[*]}; do - - # Set baseline history name - # Note that this name drops (1) the timestamp, and (2) the - # instance number for multi-instance runs - baseline_hist=${model}.${extension}.nc - - #-------------------------------------------------------------- - # Find last component hist files in this run directory, and - # determine corresponding name of the baseline file (used for - # either generation or comparison) - #-------------------------------------------------------------- - # Note that we find the last alphabetically rather than by time - # stamp, because the last by time stamp can be non-deterministic. - # Note that we need a * after ${model} to capture multi-instance - # output - - test_hist=`cd $test_dir; ls -1 ${testcase}.${model}*.${extension}.*.nc.base 2>/dev/null | tail -1` - - if [ -n "$test_hist" ]; then - - #------------------------------------------------------------- - # Do comparison, if desired - #------------------------------------------------------------- - if [ -n "$compare_tag" ]; then - compare_result=`${tools_dir}/component_compare.sh -baseline_dir "$baseline_dir" -baseline_hist "$baseline_hist" -test_dir "$test_dir" -test_hist "$test_hist" -cprnc_exe "$cprnc_exe"` - compare_status=`get_status "$compare_result"` - compare_info=`get_info "$compare_result"` - if [ "$compare_status" != "BFAIL_NA" ]; then - if [ -z "$msg" ]; then - print_comment "$compare_status" "$compare_info : $model.${extension}.nc : baseline compare ${model}.${extension} base with ${compare_tag}" "${testcase_base}" - else - print_comment "$compare_status" "$compare_info : $model.${extension}.nc : baseline compare ${model}.${extension} ($msg)" "${testcase_base}" - fi - # if ANY teststatus is FAIL then overall status is FAIL - if [ "$compare_status" = "FAIL" ]; then - overall_compare_status="FAIL" - fi - # if ANY teststatus is BFAIL then overall status is BFAIL - if [ "$overall_compare_status" != "FAIL" ]; then - if [ "$compare_status" = "BFAIL" ]; then - overall_compare_status="BFAIL" - fi - fi - fi - fi - - #-------------------------------------------------------------- - # Do baseline generation, if desired - #-------------------------------------------------------------- - if [ -n "$generate_tag" ]; then - generate_result=`${tools_dir}/component_generate.sh -baseline_dir "$baseline_dir" -baseline_hist "$baseline_hist" -test_dir "$test_dir" -test_hist "$test_hist"` - generate_status=`get_status "$generate_result"` - generate_info=`get_info "$generate_result"` - if [ "$generate_status" != "GFAIL_NA" ]; then - print_comment "$generate_status" "$generate_info : $model.${extension}.nc : baseline generate ${model}.${extension} in baseline dir ${generate_tag}" "${testcase_base}" - fi - # if ANY teststatus is GFAIL then overall status is GFAIL - if [ "$generate_status" = "GFAIL" ]; then - overall_generate_status="GFAIL" - fi - fi - fi - - done # loop over history file extensions - -done # loop over models - -if [ -n "$compare_tag" ]; then - if [ -z "$msg" ]; then - print_status "$overall_compare_status" "$compare_info : baseline compare summary" "${testcase_base}" compare - else - print_status "$overall_compare_status" "$compare_info : baseline compare summary ($msg)" "${testcase_base}" compare - fi - - if [[ $overall_compare_status == PASS ]]; then - exit 0 - else - exit 2 - fi -fi -if [ -n "$generate_tag" ]; then - print_status "$overall_generate_status" "$generate_info : baseline generate summary" "${testcase_base}" generate - - if [[ $overall_generate_status == PASS ]]; then - exit 0 - else - exit 2 - fi -fi diff --git a/scripts/Tools/component_generate.sh b/scripts/Tools/component_generate.sh deleted file mode 100755 index 1dcb0c9547b..00000000000 --- a/scripts/Tools/component_generate.sh +++ /dev/null @@ -1,238 +0,0 @@ -#!/bin/bash - -# Bill Sacks -# 8-22-12 - -#====================================================================== -# Overview -#====================================================================== - -# This script does the '-generate' step for a single test case and a -# single history file. -# -# It is intended to be used by another script, such as -# component_gen_comp. -# -# The definition of the result is similar to that in testcase_end, but -# differs slightly. For example: In testcase_end, it is a FAIL for -# baselineroot to not exist; here we allow that, and happily generate -# that directory for you if it doesn't already exist. -# -# Exit status will generally be 0 (even for test failure), but will be -# non-zero for incorrect usage. -# -# See usage message for details on inputs & return value. - -#====================================================================== -# Testing -#====================================================================== - -# There currently is no unit test script for this. Here are the unit -# tests that should be done on this script: -# -# - Missing arguments -# - return status should be "UNDEF" -# -# - test_hist='' -# - return status should be "GFAIL" -# - example: tst=`component_generate.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist ''` -# -# - success -# - return status should be "PASS" -# - example: tst=`component_generate.sh -baseline_dir /glade/scratch/sacks/cesm_baselines/test_script/ERI44y.f09_g16.TGRCP85.bluefire_ibm -baseline_hist cism.h.nc -test_dir /ptmp/sacks/ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029/run -test_hist ERI44y.f09_g16.TGRCP85.bluefire_ibm.C.114029.cism.h.2046-01-01-00000.nc` - -#====================================================================== -# Local functions -#====================================================================== - -function Usage { - echo "SYNOPSIS" - echo " $progname [options]" - echo "" - echo "RETURN VALUE" - echo " String of the format 'STATUS:Extra info about failure'" - echo " The string before the colon is the test status." - echo " The string after the colon is extra information about a failure" - echo " (this may be blank, but the colon will still be present)" - echo "" - echo " Possible values for STATUS are:" - echo " UNDEF : undefined result; this includes incorrect usage" - echo " GFAIL_NA: no history file in test case" - echo " (note that GFAIL_NA is only a problem if it occurs for all possible" - echo " history extensions for a given component)" - echo " GFAIL : error creating baseline directory or copying baseline file into place" - echo " PASS : success" - echo "" - echo "OPTIONS" - echo " -baseline_dir Full path to the baseline directory for this test (required)" - echo " -baseline_hist Name used for history file in the baseline directory (required)" - echo " -test_dir Full path to the directory containing history files for this test (required)" - echo " -test_hist Name of history file in the test directory (required)" - echo " (it is NOT an error for this to be an empty string, but this will generate a GFAIL_NA;" - echo " this will be the case when there are no history files for a component in the test directory)" - echo " -help Print this help message and exit" - echo "" - echo "EXAMPLES" - echo " TEST=SMS.T31_g37.IG4804.bluefire_ibm" - echo " CASE=\$TEST.GC.134426" - echo "" - echo " $progname -baseline_dir /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST" - echo " -baseline_hist clm.h.nc" - echo " -test_dir /glade/scratch/\$USER/\$CASE/run" - echo " -test_hist \$CASE.clm.h0.0001-12.nc" - echo " This will copy /glade/scratch/\$USER/\$CASE/run/\$CASE.clm.h0.0001-12.nc" - echo " to /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST/clm.h.nc" - echo "" - echo " $progname -baseline_dir /glade/scratch/\$USER/cesm_baselines/cesm1_1_beta16/\$TEST" - echo " -baseline_hist clm.h.nc" - echo " -test_dir /glade/scratch/\$USER/\$CASE/run" - echo " -test_hist ''" - echo " This will generate a GFAIL_NA because there is no history file to copy to the baseline directory" - echo "" -} - -# Prints the test status and an info string, separated by ':' -# Inputs: -# status: test status -# info: optional auxiliary info about test failure -function print_result { - status="$1" - info="$2" - - echo "${status}:${info}" -} - - -#====================================================================== -# Begin main script -#====================================================================== - -progname=`basename $0` - -#---------------------------------------------------------------------- -# Set default return values -#---------------------------------------------------------------------- -status='UNDEF' -info='' - -#---------------------------------------------------------------------- -# Define default values for command-line arguments -#---------------------------------------------------------------------- -baseline_dir='' -baseline_hist='' -test_dir='' - -# test_hist will often be the empty string, so we initialize it to -# something else so we can check whether it has been provided -test_hist_init='XXX_NOTGIVEN_XXX' -test_hist=$test_hist_init - -#---------------------------------------------------------------------- -# Process command-line arguments -#---------------------------------------------------------------------- -while [ $# -gt 0 ]; do - case $1 in - -baseline_dir ) - baseline_dir=$2 - shift - ;; - -baseline_hist ) - baseline_hist=$2 - shift - ;; - -test_dir ) - test_dir=$2 - shift - ;; - -test_hist ) - test_hist=$2 - shift - ;; - -help ) - Usage - exit 0 - ;; - * ) - echo "$progname: Unknown argument: $1" >&2 - echo "Run $progname -help for usage" >&2 - # return default values for status & info - print_result $status "$info" - exit 1 - ;; - esac - shift -done - - -#---------------------------------------------------------------------- -# Exit if required command-line arguments weren't provided -#---------------------------------------------------------------------- -error=0 # no errors yet - -if [ -z "$baseline_dir" ]; then - echo "$progname: baseline_dir must be provided" >&2 - error=1 -fi -if [ -z "$baseline_hist" ]; then - echo "$progname: baseline_hist must be provided" >&2 - error=1 -fi -if [ -z "$test_dir" ]; then - echo "$progname: test_dir must be provided" >&2 - error=1 -fi -if [ "$test_hist" = "$test_hist_init" ]; then - echo "$progname: test_hist must be provided" >&2 - error=1 -fi - -if [ $error -gt 0 ]; then - echo "" >&2 - echo "Run $progname -help for usage" >&2 - # return default values for status & info - print_result $status "$info" - exit 1 -fi - -#---------------------------------------------------------------------- -# Make sure there is a history file in the test case -#---------------------------------------------------------------------- -if [ -z "$test_hist" ]; then - status="GFAIL_NA" - info="no history file in test case" - print_result $status "$info" - exit 1 -fi - -#---------------------------------------------------------------------- -# Create baseline directory, if necessary -#---------------------------------------------------------------------- -mkdir -p $baseline_dir -if [ $? -ne 0 ]; then - status="GFAIL" - info="error creating baseline directory" - print_result $status "$info" - exit 1 -fi -chmod ug+w,a+rx $baseline_dir -chmod ug+w,a+rx $baseline_dir/.. - -#---------------------------------------------------------------------- -# Copy history file to baseline directory -#---------------------------------------------------------------------- -cp -f $test_dir/$test_hist $baseline_dir/$baseline_hist -if [ $? -ne 0 ]; then - status="GFAIL" - info="error copying history file to baseline directory" - print_result $status "$info" - exit 1 -fi -chmod ug+w,a+r $baseline_dir/$baseline_hist - -#---------------------------------------------------------------------- -# Return final test status -#---------------------------------------------------------------------- -status="PASS" -print_result $status "$info" -exit 0 - diff --git a/scripts/Tools/component_generate_baseline b/scripts/Tools/component_generate_baseline new file mode 100755 index 00000000000..d352760effc --- /dev/null +++ b/scripts/Tools/component_generate_baseline @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +""" +Copies current component history files into baselines +""" + +from standard_script_setup import * + +from CIME.case import Case +from CIME.hist_utils import generate_baseline + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( + usage="""\n%s [] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# Generate baselines \033[0m + > %s +""" % ((os.path.basename(args[0]), ) * 4), + description=description, + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + CIME.utils.setup_standard_logging_options(parser) + + parser.add_argument("caseroot", nargs="?", default=os.getcwd(), + help="Case directory") + + parser.add_argument("-b", "--baseline-dir", + help="Use custom baseline dir") + + args = parser.parse_args(args[1:]) + + CIME.utils.handle_standard_logging_options(args) + + return args.caseroot + +############################################################################### +def _main_func(description): +############################################################################### + caseroot, baseline_dir = parse_command_line(sys.argv, description) + with Case(caseroot) as case: + success, comments = generate_baseline(case, baseline_dir) + print comments + + sys.exit(0 if success else 1) + +if __name__ == "__main__": + _main_func(__doc__) diff --git a/utils/python/CIME/SystemTests/eri.py b/utils/python/CIME/SystemTests/eri.py index 9ae9ec92a9c..db70db99b56 100644 --- a/utils/python/CIME/SystemTests/eri.py +++ b/utils/python/CIME/SystemTests/eri.py @@ -99,8 +99,7 @@ def run_phase(self): with open("user_nl_cam", "a") as fd: fd.write("inithist = 'ENDOFRUN'\n") - self.run_indv(coupler_log_path=os.path.join(dout_sr1, "logs"), - st_archive=True) + self.run_indv(st_archive=True, suffix=None) # # (2) Test run: @@ -146,9 +145,7 @@ def run_phase(self): # run ref2 case (all component history files will go to short term archiving) - self.run_indv(suffix="hybrid", - coupler_log_path=os.path.join(dout_sr2, "logs"), - st_archive=True) + self.run_indv(suffix="hybrid", st_archive=True) # # (3a) Test run: diff --git a/utils/python/CIME/SystemTests/err.py b/utils/python/CIME/SystemTests/err.py index ffe05f956c1..dfa00ed7aa3 100644 --- a/utils/python/CIME/SystemTests/err.py +++ b/utils/python/CIME/SystemTests/err.py @@ -28,7 +28,7 @@ def run_phase(self): dout_s_root = self._case.get_value("DOUT_S_ROOT") rundir = self._case.get_value("RUNDIR") logger.info("staging files from archive %s" % dout_s_root) - for item in glob.glob(os.path.join(dout_s_root, "rest", "*", "*")): + for item in glob.glob(os.path.join(dout_s_root, "*", "hist", "*base")): shutil.copy(item, rundir) self._ers_second_phase() diff --git a/utils/python/CIME/SystemTests/ssp.py b/utils/python/CIME/SystemTests/ssp.py index e4bebfd2aeb..b4acf0a42cd 100644 --- a/utils/python/CIME/SystemTests/ssp.py +++ b/utils/python/CIME/SystemTests/ssp.py @@ -56,9 +56,7 @@ def run_phase(self): clone.flush() dout_sr = clone.get_value("DOUT_S_ROOT") - self.run_indv(suffix="spinup", - coupler_log_path=os.path.join(dout_sr, "logs"), - st_archive=True) + self.run_indv(suffix="spinup", st_archive=True) #------------------------------------------------------------------- # (2) do a hybrid, non-spinup run in orig_case diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index 19f5d2439e2..14ef1b8ce2c 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -9,6 +9,7 @@ from CIME.case_run import case_run from CIME.case_st_archive import case_st_archive from CIME.test_status import * +from CIME.hist_utils import * import CIME.build as build @@ -156,7 +157,7 @@ def _set_active_case(self, case): self._case = case self._caseroot = case.get_value("CASEROOT") - def run_indv(self, suffix="base", coupler_log_path=None, st_archive=False): + def run_indv(self, suffix="base", st_archive=False): """ Perform an individual run. Raises an EXCEPTION on fail. """ @@ -174,17 +175,18 @@ def run_indv(self, suffix="base", coupler_log_path=None, st_archive=False): logger.info(infostr) case_run(self._case) - if st_archive: - case_st_archive(self._case) - if not self._coupler_log_indicates_run_complete(coupler_log_path): + if not self._coupler_log_indicates_run_complete(): expect(False, "Coupler did not indicate run passed") if suffix is not None: self._component_compare_move(suffix) - def _coupler_log_indicates_run_complete(self, coupler_log_path): - newestcpllogfile = self._get_latest_cpl_log(coupler_log_path) + if st_archive: + case_st_archive(self._case) + + def _coupler_log_indicates_run_complete(self): + newestcpllogfile = self._get_latest_cpl_log() logger.debug("Latest Coupler log file is %s" % newestcpllogfile) # Exception is raised if the file is not compressed try: @@ -201,37 +203,20 @@ def report(self): pass def _component_compare_move(self, suffix): - cmd = os.path.join(self._case.get_value("SCRIPTSROOT"), "Tools", - "component_compare_move.sh") - rc, out, err = run_cmd("%s -rundir %s -testcase %s -suffix %s" % - (cmd, self._case.get_value('RUNDIR'), self._case.get_value('CASE'), suffix)) - if rc == 0: - append_status(out, sfile="TestStatus.log") - else: - append_status("Component_compare_move.sh failed out: %s\n\nerr: %s\n" % (out, err), - sfile="TestStatus.log") + comments = move(self._case, suffix) + append_status(comments, sfile="TestStatus.log") def _component_compare_test(self, suffix1, suffix2): """ Return value is not generally checked, but is provided in case a custom run case needs indirection based on success. """ - cmd = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", - "component_compare_test.sh") - rc, out, err = run_cmd("%s -rundir %s -testcase %s -testcase_base %s -suffix1 %s -suffix2 %s -msg 'Compare %s and %s'" - %(cmd, self._case.get_value('RUNDIR'), self._case.get_value('CASE'), - self._case.get_value('CASEBASEID'), suffix1, suffix2, suffix1, suffix2)) - logger.debug("run %s results %d %s %s"%(cmd,rc,out,err)) - status = TEST_PASS_STATUS if rc == 0 else TEST_FAIL_STATUS + success, comments = compare_test(self._case, suffix1, suffix2) + append_status(comments, sfile="TestStatus.log") + status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS with self._test_status: self._test_status.set_status("%s_%s_%s" % (COMPARE_PHASE, suffix1, suffix2), status) - - if rc != 0: - append_status("Component_compare_test.sh failed out: %s\n\nerr: %s\n"%(out,err), - sfile="TestStatus.log") - return False - - return True + return success def _get_mem_usage(self, cpllog): """ @@ -307,11 +292,11 @@ def compare_env_run(self, expected=None): return False return True - def _get_latest_cpl_log(self, coupler_log_path=None): + def _get_latest_cpl_log(self): """ find and return the latest cpl log file in the run directory """ - coupler_log_path = self._case.get_value("RUNDIR") if coupler_log_path is None else coupler_log_path + coupler_log_path = self._case.get_value("RUNDIR") cpllog = None cpllogs = glob.glob(os.path.join(coupler_log_path, 'cpl.log.*')) if cpllogs: @@ -324,27 +309,13 @@ def _compare_baseline(self): compare the current test output to a baseline result """ with self._test_status: - baselineroot = self._case.get_value("BASELINE_ROOT") - basecmp_dir = os.path.join(baselineroot, self._case.get_value("BASECMP_CASE")) - for bdir in (baselineroot, basecmp_dir): - if not os.path.isdir(bdir): - comment = "ERROR %s does not exist" % bdir - self._test_status.set_status("%s_baseline" % COMPARE_PHASE, TEST_FAIL_STATUS, comments=comment) - append_status(comment, sfile="TestStatus.log") - return -1 - - compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", - "component_compgen_baseline.sh") - compgen += " -baseline_dir "+basecmp_dir - compgen += " -test_dir "+self._case.get_value("RUNDIR") - compgen += " -compare_tag "+self._case.get_value("BASELINE_NAME_CMP") - compgen += " -testcase "+self._case.get_value("CASE") - compgen += " -testcase_base "+self._case.get_value("CASEBASEID") - rc, out, err = run_cmd(compgen) - - status = TEST_PASS_STATUS if rc == 0 else TEST_FAIL_STATUS - self._test_status.set_status("%s_baseline" % COMPARE_PHASE, status) - append_status("Baseline compare results: %s\n%s"%(out,err), sfile="TestStatus.log") + # compare baseline + success, comments = compare_baseline(self._case) + append_status(comments, sfile="TestStatus.log") + status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS + ts_comments = comments if "\n" not in comments else None + self._test_status.set_status("%s_baseline" % COMPARE_PHASE, status, comments=ts_comments) + basecmp_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASECMP_CASE")) # compare memory usage to baseline newestcpllogfile = self._get_latest_cpl_log() @@ -382,31 +353,16 @@ def _generate_baseline(self): generate a new baseline case based on the current test """ with self._test_status: - newestcpllogfile = self._get_latest_cpl_log() - baselineroot = self._case.get_value("BASELINE_ROOT") - basegen_dir = os.path.join(baselineroot, self._case.get_value("BASEGEN_CASE")) - for bdir in (baselineroot, basegen_dir): - if not os.path.isdir(bdir): - comment = "ERROR %s does not exist" % bdir - self._test_status.set_status("%s" % GENERATE_PHASE, TEST_FAIL_STATUS, comments=comment) - append_status(comment, sfile="TestStatus.log") - return -1 - - compgen = os.path.join(self._case.get_value("SCRIPTSROOT"),"Tools", - "component_compgen_baseline.sh") - compgen += " -baseline_dir "+basegen_dir - compgen += " -test_dir "+self._case.get_value("RUNDIR") - compgen += " -generate_tag "+self._case.get_value("BASELINE_NAME_GEN") - compgen += " -testcase "+self._case.get_value("CASE") - compgen += " -testcase_base "+self._case.get_value("CASEBASEID") - rc, out, err = run_cmd(compgen) - - status = TEST_PASS_STATUS if rc == 0 else TEST_FAIL_STATUS + # generate baseline + success, comments = generate_baseline(self._case) + append_status(comments, sfile="TestStatus.log") + status = TEST_PASS_STATUS if success else TEST_FAIL_STATUS self._test_status.set_status("%s" % GENERATE_PHASE, status) - append_status("Baseline generate results: %s\n%s"%(out,err), sfile="TestStatus.log") + basegen_dir = os.path.join(self._case.get_value("BASELINE_ROOT"), self._case.get_value("BASEGEN_CASE")) # copy latest cpl log to baseline # drop the date so that the name is generic + newestcpllogfile = self._get_latest_cpl_log() shutil.copyfile(newestcpllogfile, os.path.join(basegen_dir,"cpl.log.gz")) @@ -435,18 +391,18 @@ def build_phase(self, sharedlib_only=False, model_only=False): build.post_build(self._case, []) - def run_phase(self): - self.run_indv(suffix=None) - class TESTRUNPASS(FakeTest): def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") + cimeroot = self._case.get_value("CIMEROOT") + case = self._case.get_value("CASE") script = \ """ echo Insta pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID -""" % rundir +cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +""" % (rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, sharedlib_only=sharedlib_only, model_only=model_only) @@ -468,9 +424,9 @@ def build_phase(self, sharedlib_only=False, model_only=False): echo Insta pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID if [ -z "$TESTRUNDIFF_ALTERNATE" ]; then - cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc.base + cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc else - cp %s/utils/python/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc.base + cp %s/utils/python/tests/cpl.hi2.nc.test %s/%s.cpl.hi.0.nc fi """ % (rundir, cimeroot, rundir, case, cimeroot, rundir, case) self._set_script(script) @@ -501,12 +457,15 @@ class TESTRUNSLOWPASS(FakeTest): def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") + cimeroot = self._case.get_value("CIMEROOT") + case = self._case.get_value("CASE") script = \ """ sleep 300 echo Slow pass echo SUCCESSFUL TERMINATION > %s/cpl.log.$LID -""" % rundir +cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +""" % (rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, sharedlib_only=sharedlib_only, model_only=model_only) @@ -515,12 +474,14 @@ class TESTMEMLEAKFAIL(FakeTest): def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") + case = self._case.get_value("CASE") testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.failmemleak.gz") script = \ """ echo Insta pass gunzip -c %s > %s/cpl.log.$LID -""" % (testfile, rundir) +cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +""" % (testfile, rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, sharedlib_only=sharedlib_only, model_only=model_only) @@ -529,12 +490,14 @@ class TESTMEMLEAKPASS(FakeTest): def build_phase(self, sharedlib_only=False, model_only=False): rundir = self._case.get_value("RUNDIR") cimeroot = self._case.get_value("CIMEROOT") + case = self._case.get_value("CASE") testfile = os.path.join(cimeroot,"utils","python","tests","cpl.log.passmemleak.gz") script = \ """ echo Insta pass gunzip -c %s > %s/cpl.log.$LID -""" % (testfile, rundir) +cp %s/utils/python/tests/cpl.hi1.nc.test %s/%s.cpl.hi.0.nc +""" % (testfile, rundir, cimeroot, rundir, case) self._set_script(script) FakeTest.build_phase(self, sharedlib_only=sharedlib_only, model_only=model_only) diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py index d398b558894..de651fea728 100644 --- a/utils/python/CIME/bless_test_results.py +++ b/utils/python/CIME/bless_test_results.py @@ -1,85 +1,50 @@ import CIME.compare_namelists, CIME.simple_compare from CIME.test_scheduler import NAMELIST_PHASE -from CIME.utils import run_cmd, run_cmd_no_fail, expect +from CIME.utils import run_cmd, expect from CIME.XML.machines import Machines from CIME.test_status import * +from CIME.hist_utils import generate_baseline, compare_baseline +from CIME.case import Case import os, glob, time _MACHINE = Machines() ############################################################################### -def bless_namelists(test_name, baseline_dir_for_test, testcase_dir_for_test, report_only, force): +def bless_namelists(test_name, report_only, force, baseline_name): ############################################################################### - namelist_files = [] - - for root, _, files in os.walk(baseline_dir_for_test): - if (root == baseline_dir_for_test): - rel_root = "" - else: - rel_root = root.replace("%s/" % baseline_dir_for_test, "") - - for file_ in files: - rel_file = os.path.join(rel_root, file_) - - baseline_file = os.path.join(baseline_dir_for_test, rel_file) - testcase_file = os.path.join(testcase_dir_for_test, rel_file) - - if (os.path.isfile(testcase_file) and (rel_root == "CaseDocs" or file_.startswith("user_nl"))): - if (CIME.compare_namelists.is_namelist_file(baseline_file)): - if (not CIME.compare_namelists.compare_namelist_files(baseline_file, testcase_file, test_name)): - print "Namelist files '%s' and '%s' did not match" % (baseline_file, testcase_file) - print - if (not report_only and - (force or raw_input("Update this file (y/n)? ").upper() in ["Y", "YES"])): - namelist_files.append((rel_file, rel_file)) - else: - if (not CIME.simple_compare.compare_files(baseline_file, testcase_file, test_name)): - print "Simple files '%s' and '%s' did not match" % (baseline_file, testcase_file) - print - if (not report_only and - (force or raw_input("Update this file (y/n)? ").upper() in ["Y", "YES"])): - namelist_files.append((rel_file, rel_file)) - # Update namelist files - if (namelist_files): - CIME.utils.safe_copy(testcase_dir_for_test, baseline_dir_for_test, namelist_files) + print "Test '%s' had a namelist diff" % test_name + if (not report_only and + (force or raw_input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): + stat, _, err = run_cmd("create_test -n -g %s -b %s" % (test_name, baseline_name)) + if stat != 0: + return False, "Namelist regen failed: '%s'" % err + else: + return True, None ############################################################################### -def bless_history(test_name, baseline_tag, baseline_dir_for_test, testcase_dir_for_test, report_only, force): +def bless_history(test_name, testcase_dir_for_test, baseline_name, report_only, force): ############################################################################### - # Get user that test was run as (affects loc of hist files) - acme_root = run_cmd_no_fail("./xmlquery CESMSCRATCHROOT -value", from_dir=testcase_dir_for_test) - - case = os.path.basename(testcase_dir_for_test) - cime_root = CIME.utils.get_cime_root() - compgen = os.path.join(cime_root, "scripts", "Tools", "component_compgen_baseline.sh") - machine_env = os.path.join(testcase_dir_for_test, ".env_mach_specific.sh") - run_dir = os.path.join(acme_root, case, "run") - cprnc_loc = _MACHINE.get_value("CCSM_CPRNC") - compgen_cmd = "source %s && %s -baseline_dir %s -testcase %s -testcase_base %s -test_dir %s -cprnc_exe %s -generate_tag %s" % \ - (machine_env, compgen, baseline_dir_for_test, case, test_name, run_dir, cprnc_loc, baseline_tag) - - check_compare = os.path.join(cime_root, "scripts", "Tools", "component_write_comparefail.pl") - stat, out, _ = run_cmd("%s %s 2>&1" % (check_compare, run_dir)) - - if (stat != 0): - # found diff, offer rebless - print out - - if (not report_only and - (force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): - stat = run_cmd(compgen_cmd, verbose=True)[0] - if (stat != 0): - logging.warning("Hist file bless FAILED for test %s" % test_name) - return False, "Bless command failed" + with Case(testcase_dir_for_test) as case: + baseline_full_dir = os.path.join(case.get_value("BASELINE_ROOT"), case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) + result, comments = compare_baseline(case, baseline_dir=baseline_full_dir) + if result: + logging.warning("Test '%s' was marked as DIFF but compare_baseline did not find diff?" % test_name) + return False, "No diff found" + else: + print comments + if (not report_only and + (force or raw_input("Update this diff (y/n)? ").upper() in ["Y", "YES"])): + result, comments = generate_baseline(case, baseline_dir=baseline_full_dir) + if not result: + logging.warning("Hist file bless FAILED for test %s" % test_name) + return False, "Generate baseline failed: %s" % comments + else: + print comments + return True, None else: return True, None - else: - return True, None - else: - logging.warning("Test '%s' was marked as DIFF but cprnc did not find diff?" % test_name) - return False, "No diff found or missing baseline" ############################################################################### def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelists_only=False, hist_only=False, report_only=False, force=False, bless_tests=None): @@ -88,10 +53,6 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) expect(test_status_files, "No matching test cases found in for %s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) - baseline_root = _MACHINE.get_value("CCSM_BASELINE") - baseline_tag = os.path.join(compiler, baseline_name) - baseline_area = os.path.join(baseline_root, baseline_tag) - # The env_mach_specific script may need these to be defined compiler = _MACHINE.get_default_compiler() # this MUST match compiler that cprnc was built with os.environ["COMPILER"] = compiler @@ -147,13 +108,6 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis print "###############################################################################" time.sleep(2) - # Get baseline dir for this test - baseline_dir_for_test = os.path.join(baseline_area, test_name) - if (not os.path.isdir(baseline_dir_for_test)): - logging.warning("Problem, baseline dir '%s' does not exist" % baseline_dir_for_test) - broken_blesses.append((test_name, "missing baseline dir")) - continue - # Get testcase dir for this test if (test_id is None): # The full name already contains the compiler, so we just need to glob for the branch name @@ -170,11 +124,13 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis # Bless namelists if (not nl_no_bless): - bless_namelists(test_name, baseline_dir_for_test, testcase_dir_for_test, report_only, force) + success, reason = bless_namelists(test_name, report_only, force, baseline_name) + if not success: + broken_blesses.append(test_name, reason) # Bless hist files if (not hist_no_bless): - success, reason = bless_history(test_name, baseline_tag, baseline_dir_for_test, testcase_dir_for_test, report_only, force) + success, reason = bless_history(test_name, testcase_dir_for_test, baseline_name, report_only, force) if (not success): broken_blesses.append((test_name, reason)) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py new file mode 100644 index 00000000000..b59c1a2fbd2 --- /dev/null +++ b/utils/python/CIME/hist_utils.py @@ -0,0 +1,172 @@ +""" +Functions for actions pertaining to history files. +""" + +from CIME.XML.standard_module_setup import * +from CIME.XML.component import Component + +import logging, glob, os, shutil, re + +def _iter_model_file_substrs(): + drv_comp = Component() + models = drv_comp.get_valid_model_components() + + for model in models: + model = "cpl" if model == "DRV" else model.lower() + yield model + +def _get_all_hist_files(testcase, model, suffix="", from_dir=os.getcwd()): + suffix = (".%s" % suffix) if suffix else "" + test_hists = glob.glob("%s/%s.%s*.h?.nc%s" % (from_dir, testcase, model, suffix)) + test_hists.extend(glob.glob("%s/%s.%s*.h.nc%s" % (from_dir, testcase, model, suffix))) + test_hists.extend(glob.glob("%s/%s.%s*.h?.*.nc%s" % (from_dir, testcase, model, suffix))) + test_hists.extend(glob.glob("%s/%s.%s*.h.*.nc%s" % (from_dir, testcase, model, suffix))) + if suffix == "": + test_hists.extend(glob.glob("%s/%s.h.nc" % (from_dir, model))) + test_hists.extend(glob.glob("%s/%s.h?.nc" % (from_dir, model))) + test_hists.sort() + return test_hists + +def move(case, suffix): + """ + Change the suffix for the most recent batch of hist files + """ + rundir = case.get_value("RUNDIR") + testcase = case.get_value("CASE") + + # Loop over models + comments = "Moving hist files to suffix '%s'\n" % suffix + num_moved = 0 + for model in _iter_model_file_substrs(): + comments += " Moving hist files for model '%s'\n" % model + test_hists = _get_all_hist_files(testcase, model, from_dir=rundir) + num_moved += len(test_hists) + for test_hist in test_hists: + new_file = "%s.%s" % (test_hist, suffix) + if os.path.exists(new_file): + os.remove(new_file) + + comments += " Copying '%s' to '%s'\n" % (test_hist, new_file) + shutil.copy(test_hist, new_file) + + expect(num_moved > 0, "move failed: no hist files found in rundir '%s'" % rundir) + + return comments + +def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir2=os.getcwd()): + + testcase = case.get_value("CASE") + all_success = True + num_compared = 0 + comments = "Comparing hists for case '%s' dir1='%s', suffix1='%s', dir2='%s' suffix2='%s'\n" % \ + (testcase, from_dir1, suffix1, from_dir2, suffix2) + + for model in _iter_model_file_substrs(): + comments += " comparing model '%s'\n" % model + hists1 = _get_all_hist_files(testcase, model, suffix1, from_dir1) + hists2 = _get_all_hist_files(testcase, model, suffix2, from_dir2) + if len(hists1) != len(hists2): + comments += " num hists does not match %d != %d\n" % (len(hists1), len(hists2)) + all_success = False + continue + + num_compared += len(hists1) + for hist1, hist2 in zip(hists1, hists2): + success, cprnc_comments = cprnc(hist1, hist2, case, from_dir1) + if success: + comments += " %s matched %s\n" % (hist1, hist2) + else: + comments += " %s did NOT match %s\n" % (hist1, hist2) + comments += cprnc_comments + "\n" + all_success = False + + expect(num_compared > 0, "Did not compare any hist files for suffix1='%s' suffix2='%s', dir1='%s', dir2='%s'\nComments=%s" % + (suffix1, suffix2, from_dir1, from_dir2, comments)) + return all_success, comments + +def compare_test(case, suffix1, suffix2): + """ + Compares two component history files in the testcase directory + + returns (SUCCESS, comments) + """ + rundir = case.get_value("RUNDIR") + + return _compare_hists(case, suffix1, suffix2, rundir, rundir) + +def cprnc(file1, file2, case, rundir): + cprnc_exe = case.get_value("CCSM_CPRNC") + basename = os.path.basename(file1) + stat, out, _ = run_cmd("%s %s %s 2>&1 | tee %s/%s.cprnc.out" % (cprnc_exe, file1, file2, rundir, basename)) + return (stat == 0 and "IDENTICAL" in out, out) + +def compare_baseline(case, baseline_dir=None): + """ + compare the current test output to a baseline result + """ + rundir = case.get_value("RUNDIR") + if baseline_dir is None: + baselineroot = case.get_value("BASELINE_ROOT") + basecmp_dir = os.path.join(baselineroot, case.get_value("BASECMP_CASE")) + dirs_to_check = (baselineroot, basecmp_dir) + else: + basecmp_dir = baseline_dir + dirs_to_check = (basecmp_dir,) + + for bdir in dirs_to_check: + if not os.path.isdir(bdir): + return False, "ERROR %s does not exist" % bdir + + return _compare_hists(case, from_dir1=rundir, from_dir2=basecmp_dir) + +def get_extension(model, filepath): + """ + >>> get_extension("cpl", "cpl.hi.nc") + 'hi' + >>> get_extension("cpl", "cpl.h.nc") + 'h' + >>> get_extension("cpl", "cpl.h1.nc") + 'h1' + >>> get_extension("cpl", "TESTRUNDIFF_Mmpi-serial.f19_g16_rx1.A.melvin_gnu.C.fake_testing_only_20160816_164150-20160816_164240.cpl.hi.0.nc.base") + 'hi' + >>> get_extension("cpl", "TESTRUNDIFF_Mmpi-serial.f19_g16_rx1.A.melvin_gnu.C.fake_testing_only_20160816_164150-20160816_164240.cpl.h.nc") + 'h' + """ + basename = os.path.basename(filepath) + ext_regex = re.compile(r'.*%s.*[.](h.?)([.][^.]+)?[.]nc' % model) + m = ext_regex.match(basename) + expect(m is not None, "Failed to get extension for file '%s'" % filepath) + return m.groups()[0] + +def generate_baseline(case, baseline_dir=None): + """ + copy the current test output to baseline result + """ + rundir = case.get_value("RUNDIR") + if baseline_dir is None: + baselineroot = case.get_value("BASELINE_ROOT") + basegen_dir = os.path.join(baselineroot, case.get_value("BASEGEN_CASE")) + else: + basegen_dir = baseline_dir + testcase = case.get_value("CASE") + + if not os.path.isdir(basegen_dir): + os.makedirs(basegen_dir) + + comments = "Generating baselines into '%s'\n" % basegen_dir + num_gen = 0 + for model in _iter_model_file_substrs(): + comments += " generating for model '%s'\n" % model + hists = _get_all_hist_files(testcase, model, from_dir=rundir) + num_gen += len(hists) + for hist in hists: + ext = get_extension(model, hist) + basename = "%s.%s.nc" % (model, ext) + baseline = os.path.join(basegen_dir, basename) + if os.path.exists(baseline): + os.remove(baseline) + + shutil.copy(hist, baseline) + comments += " generating baseline '%s'\n" % baseline + + return True, comments diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py index 842e739ddee..8f45dd82b2c 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/utils/python/tests/scripts_regression_tests.py @@ -161,6 +161,7 @@ def tearDown(self): print "Leaving case directory : %s"%tfile elif do_teardown: shutil.rmtree(tfile) + ############################################################################### class D_TestWaitForTests(unittest.TestCase): ############################################################################### @@ -578,7 +579,7 @@ def test_b_full(self): ts = TestStatus(test_dir=os.path.dirname(test_status)) test_name = ts.get_name() log_files = glob.glob("%s/%s*%s/TestStatus.log" % (self._testroot, test_name, test_id)) - self.assertEqual(len(log_files), 1, "Expected exactly one TestStatus.log file, foudn %d" % len(log_files)) + self.assertEqual(len(log_files), 1, "Expected exactly one TestStatus.log file, found %d" % len(log_files)) log_file = log_files[0] if (test_name == build_fail_test): self.assertEqual(ts.get_status(CIME.test_scheduler.MODEL_BUILD_PHASE), TEST_FAIL_STATUS) From cc502e4ce7fb70bf0bb684c41c79e664cda15556 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 17 Aug 2016 13:50:19 -0600 Subject: [PATCH 02/20] Fix mistake caught by code review --- scripts/Tools/component_compare_baseline | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Tools/component_compare_baseline b/scripts/Tools/component_compare_baseline index f6e25bd0d62..ea8e43343a7 100755 --- a/scripts/Tools/component_compare_baseline +++ b/scripts/Tools/component_compare_baseline @@ -39,7 +39,7 @@ OR CIME.utils.handle_standard_logging_options(args) - return args.caseroot + return args.caseroot, args.baseline_dir ############################################################################### def _main_func(description): From 39a778b4b335ee970e8aac4a4bf276e2e6886f78 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 17 Aug 2016 16:46:02 -0600 Subject: [PATCH 03/20] fixes in hist_utils --- utils/python/CIME/hist_utils.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index b59c1a2fbd2..b0e444a89ae 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -3,16 +3,14 @@ """ from CIME.XML.standard_module_setup import * -from CIME.XML.component import Component import logging, glob, os, shutil, re -def _iter_model_file_substrs(): - drv_comp = Component() - models = drv_comp.get_valid_model_components() +def _iter_model_file_substrs(case): + models = case.get_compset_components() + models.append('cpl') for model in models: - model = "cpl" if model == "DRV" else model.lower() yield model def _get_all_hist_files(testcase, model, suffix="", from_dir=os.getcwd()): @@ -37,9 +35,11 @@ def move(case, suffix): # Loop over models comments = "Moving hist files to suffix '%s'\n" % suffix num_moved = 0 - for model in _iter_model_file_substrs(): + for model in _iter_model_file_substrs(case): comments += " Moving hist files for model '%s'\n" % model test_hists = _get_all_hist_files(testcase, model, from_dir=rundir) + if test_hists is None: + continue num_moved += len(test_hists) for test_hist in test_hists: new_file = "%s.%s" % (test_hist, suffix) @@ -61,12 +61,17 @@ def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir comments = "Comparing hists for case '%s' dir1='%s', suffix1='%s', dir2='%s' suffix2='%s'\n" % \ (testcase, from_dir1, suffix1, from_dir2, suffix2) - for model in _iter_model_file_substrs(): + for model in _iter_model_file_substrs(case): comments += " comparing model '%s'\n" % model hists1 = _get_all_hist_files(testcase, model, suffix1, from_dir1) hists2 = _get_all_hist_files(testcase, model, suffix2, from_dir2) - if len(hists1) != len(hists2): - comments += " num hists does not match %d != %d\n" % (len(hists1), len(hists2)) + len_hist1 = 0 if hist1 is None else len(hists1) + len_hist2 = 0 if hist2 is None else len(hists2) + if len_hist1 == 0: + comments += " no hist files found for model %s\n"%model + continue + if len_hist1 != len_hist2: + comments += " num hists does not match %d != %d\n" % (len_hist1, len_hist2) all_success = False continue @@ -97,8 +102,8 @@ def compare_test(case, suffix1, suffix2): def cprnc(file1, file2, case, rundir): cprnc_exe = case.get_value("CCSM_CPRNC") basename = os.path.basename(file1) - stat, out, _ = run_cmd("%s %s %s 2>&1 | tee %s/%s.cprnc.out" % (cprnc_exe, file1, file2, rundir, basename)) - return (stat == 0 and "IDENTICAL" in out, out) + stat, out, _ = run_cmd("%s -m %s %s 2>&1 | tee %s/%s.cprnc.out" % (cprnc_exe, file1, file2, rundir, basename)) + return (stat == 0 and "files seem to be IDENTICAL" in out, out) def compare_baseline(case, baseline_dir=None): """ @@ -155,9 +160,11 @@ def generate_baseline(case, baseline_dir=None): comments = "Generating baselines into '%s'\n" % basegen_dir num_gen = 0 - for model in _iter_model_file_substrs(): + for model in _iter_model_file_substrs(case): comments += " generating for model '%s'\n" % model hists = _get_all_hist_files(testcase, model, from_dir=rundir) + if hists is None: + continue num_gen += len(hists) for hist in hists: ext = get_extension(model, hist) From 30a48da71fb403ed17ce01cd0853a5035b36f603 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Wed, 17 Aug 2016 17:17:09 -0600 Subject: [PATCH 04/20] remove check for None --- utils/python/CIME/hist_utils.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index b0e444a89ae..064633689a3 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -38,8 +38,6 @@ def move(case, suffix): for model in _iter_model_file_substrs(case): comments += " Moving hist files for model '%s'\n" % model test_hists = _get_all_hist_files(testcase, model, from_dir=rundir) - if test_hists is None: - continue num_moved += len(test_hists) for test_hist in test_hists: new_file = "%s.%s" % (test_hist, suffix) @@ -65,8 +63,8 @@ def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir comments += " comparing model '%s'\n" % model hists1 = _get_all_hist_files(testcase, model, suffix1, from_dir1) hists2 = _get_all_hist_files(testcase, model, suffix2, from_dir2) - len_hist1 = 0 if hist1 is None else len(hists1) - len_hist2 = 0 if hist2 is None else len(hists2) + len_hist1 = len(hists1) + len_hist2 = len(hists2) if len_hist1 == 0: comments += " no hist files found for model %s\n"%model continue @@ -163,8 +161,6 @@ def generate_baseline(case, baseline_dir=None): for model in _iter_model_file_substrs(case): comments += " generating for model '%s'\n" % model hists = _get_all_hist_files(testcase, model, from_dir=rundir) - if hists is None: - continue num_gen += len(hists) for hist in hists: ext = get_extension(model, hist) From 26345c090978f501b68bea83f1c2d13f7d23a3b7 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 17 Aug 2016 17:13:30 -0600 Subject: [PATCH 05/20] bless_test_results: Need sane error code Return non-zero if there were significant problems during bless --- scripts/Tools/bless_test_results | 6 +++--- utils/python/CIME/bless_test_results.py | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results index 8700184c1bb..fcfbdaa1c87 100755 --- a/scripts/Tools/bless_test_results +++ b/scripts/Tools/bless_test_results @@ -6,8 +6,7 @@ changes, and updating baselines. Purpose is, instead of re-running tests in generate mode, which is very slow, allow for very fast analsis and blessing of diffs. -Be aware that restart test will overwrite the original namelist files -with versions of the files that you should not bless. +You may need to load modules for cprnc to work. """ from standard_script_setup import * @@ -103,7 +102,8 @@ def _main_func(description): baseline_name, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests = \ parse_command_line(sys.argv, description) - CIME.bless_test_results.bless_test_results(baseline_name, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests) + success = CIME.bless_test_results.bless_test_results(baseline_name, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests) + sys.exit(0 if success else 1) ############################################################################### diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py index de651fea728..a4158291675 100644 --- a/utils/python/CIME/bless_test_results.py +++ b/utils/python/CIME/bless_test_results.py @@ -13,6 +13,10 @@ ############################################################################### def bless_namelists(test_name, report_only, force, baseline_name): ############################################################################### + # Be aware that restart test will overwrite the original namelist files + # with versions of the files that should not be blessed. This forces us to + # re-run create_test. + # Update namelist files print "Test '%s' had a namelist diff" % test_name if (not report_only and @@ -53,11 +57,6 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) expect(test_status_files, "No matching test cases found in for %s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) - # The env_mach_specific script may need these to be defined - compiler = _MACHINE.get_default_compiler() # this MUST match compiler that cprnc was built with - os.environ["COMPILER"] = compiler - os.environ["MPILIB"] = _MACHINE.get_default_MPIlib(attributes={"compiler":compiler}) - broken_blesses = [] for test_status_file in test_status_files: ts = TestStatus(test_dir=os.path.dirname(test_status_file)) @@ -106,7 +105,8 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis print "###############################################################################" print "Blessing results for test:", test_name, "most recent result:", overall_result print "###############################################################################" - time.sleep(2) + if not force: + time.sleep(2) # Get testcase dir for this test if (test_id is None): @@ -135,5 +135,9 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis broken_blesses.append((test_name, reason)) # Make sure user knows that some tests were not blessed + success = True for broken_bless, reason in broken_blesses: logging.warning("FAILED TO BLESS TEST: %s, reason %s" % (broken_bless, reason)) + success = False + + return success From a0e010e98479e75fe7ba2104680aa8b68c37d4d9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Wed, 17 Aug 2016 17:14:10 -0600 Subject: [PATCH 06/20] Add new compare_test_results, counterpart to bless_test_results Add small test for this in scripts_regression too. --- scripts/Tools/compare_test_results | 89 +++++++++++++++++++ utils/python/CIME/compare_test_results.py | 86 ++++++++++++++++++ .../python/tests/scripts_regression_tests.py | 11 +++ 3 files changed, 186 insertions(+) create mode 100755 scripts/Tools/compare_test_results create mode 100644 utils/python/CIME/compare_test_results.py diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results new file mode 100755 index 00000000000..67c54ce053b --- /dev/null +++ b/scripts/Tools/compare_test_results @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +""" +Analyze results from a test root area, comparing non-BFB +changes. Purpose is, instead of re-running tests +in generate mode, which is very slow, allow for very fast analsis of diffs. + +You may need to load modules for cprnc to work. +""" + +from standard_script_setup import * + +from CIME.utils import expect +from CIME.XML.machines import Machines +from CIME.compare_test_results import compare_test_results + +import argparse, sys, os, doctest + +_MACHINE = Machines() + +############################################################################### +def parse_command_line(args, description): +############################################################################### + parser = argparse.ArgumentParser( +usage="""\n%s [-n] [-r ] [-b ] [-c ] [ ...] [--verbose] +OR +%s --help +OR +%s --test + +\033[1mEXAMPLES:\033[0m + \033[1;32m# From most recent run, compare all changes \033[0m + > %s + \033[1;32m# From most recent run, compare only changes for test foo and bar only \033[0m + > %s -n foo bar + \033[1;32m# From most recent run of jenkins, compare history changes for next \033[0m + > %s -r /home/jenkins/acme/scratch/jenkins -b next +""" % ((os.path.basename(args[0]), ) * 6), + +description=description, + +formatter_class=argparse.ArgumentDefaultsHelpFormatter +) + + default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root()) + default_compiler = _MACHINE.get_default_compiler() + scratch_root = _MACHINE.get_value("CESMSCRATCHROOT") + default_testroot = os.path.join(scratch_root) + + CIME.utils.setup_standard_logging_options(parser) + + parser.add_argument("-b", "--baseline-name", default=default_baseline_name, + help="Name of baselines to use, corresponds to branch used.") + + parser.add_argument("-c", "--compiler", default=default_compiler, + help="Compiler of run you want to compare") + + parser.add_argument("-r", "--test-root", default=default_testroot, + help="Path to test results that are being compared") + + parser.add_argument("-t", "--test-id", + help="Limit processes to case dirs matching this test-id. Can be useful if mutiple runs dumped into the same dir.") + + parser.add_argument("compare_tests", nargs="*", + help="When comparing, limit the comparison to tests matching these regex") + + args = parser.parse_args(args[1:]) + + CIME.utils.handle_standard_logging_options(args) + + return args.baseline_name, args.test_root, args.compiler, args.test_id, args.compare_tests + +############################################################################### +def _main_func(description): +############################################################################### + if ("--test" in sys.argv): + test_results = doctest.testmod(verbose=True) + sys.exit(1 if test_results.failed > 0 else 0) + + baseline_name, test_root, compiler, test_id, compare_tests = \ + parse_command_line(sys.argv, description) + + success = CIME.compare_test_results.compare_test_results(baseline_name, test_root, compiler, test_id, compare_tests) + sys.exit(0 if success else 1) + +############################################################################### + +if (__name__ == "__main__"): + _main_func(__doc__) diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py new file mode 100644 index 00000000000..31957563dcc --- /dev/null +++ b/utils/python/CIME/compare_test_results.py @@ -0,0 +1,86 @@ +import CIME.compare_namelists, CIME.simple_compare +from CIME.utils import expect +from CIME.XML.machines import Machines +from CIME.test_status import * +from CIME.hist_utils import compare_baseline +from CIME.case import Case + +import os, glob + +_MACHINE = Machines() + +############################################################################### +def compare_history(testcase_dir_for_test, baseline_name): +############################################################################### + with Case(testcase_dir_for_test) as case: + baseline_full_dir = os.path.join(case.get_value("BASELINE_ROOT"), case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) + result, comments = compare_baseline(case, baseline_dir=baseline_full_dir) + if result: + return True, None + else: + logging.info(comments) + return False, "Diff'd" + +############################################################################### +def compare_test_results(baseline_name, test_root, compiler, test_id=None, compare_tests=None): +############################################################################### + test_id_glob = "*%s*%s*" % (compiler, baseline_name) if test_id is None else "*%s" % test_id + test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) + expect(test_status_files, "No matching test cases found in for %s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) + + broken_compares = [] + for test_status_file in test_status_files: + ts = TestStatus(test_dir=os.path.dirname(test_status_file)) + test_name = ts.get_name() + if (compare_tests in [[], None] or CIME.utils.match_any(test_name, compare_tests)): + overall_result = ts.get_overall_test_status() + + # Compute hist status, False implies it diffed + run_result = ts.get_status(RUN_PHASE) + if (run_result is None): + broken_compares.append((test_name, "no run phase")) + logging.warning("Test '%s' did not make it to run phase" % test_name) + hist_no_compare = True + elif (run_result != TEST_PASS_STATUS): + broken_compares.append((test_name, "test did not pass")) + logging.warning("Test '%s' did not pass, not safe to compare" % test_name) + hist_no_compare = True + else: + hist_no_compare = False + + # Now, do the compare + if hist_no_compare: + logging.info("Cannot compare test: %s, overall status: %s" % (test_name, overall_result)) + else: + + logging.info("###############################################################################") + logging.info("Comparing results for test: %s, most recent result: %s" % (test_name, overall_result)) + logging.info("###############################################################################") + + # Get testcase dir for this test + if (test_id is None): + # The full name already contains the compiler, so we just need to glob for the branch name + globs = glob.glob("%s/%s*%s*" % (test_root, test_name, baseline_name)) + else: + globs = glob.glob("%s/%s%s" % (test_root, test_name, test_id_glob)) + + if (len(globs) != 1): + logging.warning("Expected exactly one match for testcase area for test '%s', found '%s'" % (test_name, globs)) + broken_compares.append((test_name, "multiple matching testcase dirs")) + continue + + testcase_dir_for_test = globs[0] + + # Compare hist files + if (not hist_no_compare): + success, reason = compare_history(testcase_dir_for_test, baseline_name) + if (not success): + broken_compares.append((test_name, reason)) + + # Make sure user knows that some tests were not compareed + success = True + for broken_compare, reason in broken_compares: + logging.warning("COMPARE FAILED FOR TEST: %s, reason %s" % (broken_compare, reason)) + success = False + + return success diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py index 8f45dd82b2c..e34f1874b47 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/utils/python/tests/scripts_regression_tests.py @@ -741,6 +741,17 @@ def test_bless_test_results(self): test_id = "%s-%s" % (self._baseline_name, CIME.utils.get_utc_timestamp()) self.simple_test(False, "-c -b %s -t %s" % (self._baseline_name, test_id)) + # compare_test_results should detect the fail + cpr_cmd = "%s/compare_test_results -b %s -t %s 2>&1" % (TOOLS_DIR, self._baseline_name, test_id) + stat, out, _ = run_cmd(cpr_cmd) + self.assertNotEqual(stat, 0, msg="COMMAND '%s' should not have worked. Out:\n%s" % (cpr_cmd, out)) + + # use regex + expected_pattern = re.compile(r'COMPARE FAILED FOR TEST: %s[^\s]* reason Diff' % self._test_name) + the_match = expected_pattern.search(out) + self.assertNotEqual(the_match, None, + msg="Failed to display failed test in output:\n%s" % out) + # Bless run_cmd_no_fail("%s/bless_test_results --hist-only --force -b %s -t %s" % (TOOLS_DIR, self._baseline_name, test_id)) From 9b6943b8f5e3c4c9bf2417af17937527774a48db Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 11:47:55 -0600 Subject: [PATCH 07/20] Remove dangerous cwd defaults, add documentation to hist_utils public API --- utils/python/CIME/check_lockedfiles.py | 3 +- utils/python/CIME/hist_utils.py | 56 ++++++++++++++++++++++---- utils/python/CIME/test_status.py | 3 +- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/utils/python/CIME/check_lockedfiles.py b/utils/python/CIME/check_lockedfiles.py index 5beee69dd47..81d57977119 100644 --- a/utils/python/CIME/check_lockedfiles.py +++ b/utils/python/CIME/check_lockedfiles.py @@ -9,10 +9,11 @@ import glob -def check_lockedfiles(caseroot=os.getcwd()): +def check_lockedfiles(caseroot=None): """ Check that all lockedfiles match what's in case """ + caseroot = os.getcwd() if caseroot is None else caseroot lockedfiles = glob.glob(os.path.join(caseroot, "LockedFiles", "*.xml")) for lfile in lockedfiles: fpart = os.path.basename(lfile) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 064633689a3..33b814b88ec 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -13,21 +13,33 @@ def _iter_model_file_substrs(case): for model in models: yield model -def _get_all_hist_files(testcase, model, suffix="", from_dir=os.getcwd()): +def _get_all_hist_files(testcase, model, from_dir, suffix=""): suffix = (".%s" % suffix) if suffix else "" + + # Match hist files produced by run test_hists = glob.glob("%s/%s.%s*.h?.nc%s" % (from_dir, testcase, model, suffix)) test_hists.extend(glob.glob("%s/%s.%s*.h.nc%s" % (from_dir, testcase, model, suffix))) + + # Match multi-instance files produced by run test_hists.extend(glob.glob("%s/%s.%s*.h?.*.nc%s" % (from_dir, testcase, model, suffix))) test_hists.extend(glob.glob("%s/%s.%s*.h.*.nc%s" % (from_dir, testcase, model, suffix))) + + # suffix == "" implies baseline comparison, baseline hist files have simpler names if suffix == "": test_hists.extend(glob.glob("%s/%s.h.nc" % (from_dir, model))) test_hists.extend(glob.glob("%s/%s.h?.nc" % (from_dir, model))) + test_hists.sort() return test_hists def move(case, suffix): """ - Change the suffix for the most recent batch of hist files + Change the suffix for the most recent batch of hist files in a case. This can + allow you to temporarily "save" these files so they won't be blown away if you + re-run the case. + + case - The case containing the files you want to save + suffix - The string suffix you want to add to saved files, this can be used to find them later. """ rundir = case.get_value("RUNDIR") testcase = case.get_value("CASE") @@ -37,7 +49,7 @@ def move(case, suffix): num_moved = 0 for model in _iter_model_file_substrs(case): comments += " Moving hist files for model '%s'\n" % model - test_hists = _get_all_hist_files(testcase, model, from_dir=rundir) + test_hists = _get_all_hist_files(testcase, model, rundir) num_moved += len(test_hists) for test_hist in test_hists: new_file = "%s.%s" % (test_hist, suffix) @@ -61,8 +73,8 @@ def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir for model in _iter_model_file_substrs(case): comments += " comparing model '%s'\n" % model - hists1 = _get_all_hist_files(testcase, model, suffix1, from_dir1) - hists2 = _get_all_hist_files(testcase, model, suffix2, from_dir2) + hists1 = _get_all_hist_files(testcase, model, from_dir1, suffix1) + hists2 = _get_all_hist_files(testcase, model, from_dir2, suffix2) len_hist1 = len(hists1) len_hist2 = len(hists2) if len_hist1 == 0: @@ -89,7 +101,11 @@ def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir def compare_test(case, suffix1, suffix2): """ - Compares two component history files in the testcase directory + Compares two sets of component history files in the testcase directory + + case - The case containing the hist files to compare + suffix1 - The suffix that identifies the first batch of hist files + suffix1 - The suffix that identifies the second batch of hist files returns (SUCCESS, comments) """ @@ -98,6 +114,16 @@ def compare_test(case, suffix1, suffix2): return _compare_hists(case, suffix1, suffix2, rundir, rundir) def cprnc(file1, file2, case, rundir): + """ + Run cprnc to compare two individual nc files + + file1 - the full or relative path of the first file + file2 - the full or relative path of the second file + case - the case containing the files + rundir - the rundir for the case + + returns True if the files matched + """ cprnc_exe = case.get_value("CCSM_CPRNC") basename = os.path.basename(file1) stat, out, _ = run_cmd("%s -m %s %s 2>&1 | tee %s/%s.cprnc.out" % (cprnc_exe, file1, file2, rundir, basename)) @@ -106,6 +132,12 @@ def cprnc(file1, file2, case, rundir): def compare_baseline(case, baseline_dir=None): """ compare the current test output to a baseline result + + case - The case containing the hist files to be compared against baselines + baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config + + returns (SUCCESS, comments) + SUCCESS means all hist files matched their corresponding baseline """ rundir = case.get_value("RUNDIR") if baseline_dir is None: @@ -124,6 +156,11 @@ def compare_baseline(case, baseline_dir=None): def get_extension(model, filepath): """ + For a hist file for the given model, return what we call the "extension" + + model - The component model + filepath - The path of the hist file + >>> get_extension("cpl", "cpl.hi.nc") 'hi' >>> get_extension("cpl", "cpl.h.nc") @@ -144,6 +181,11 @@ def get_extension(model, filepath): def generate_baseline(case, baseline_dir=None): """ copy the current test output to baseline result + + case - The case containing the hist files to be copied into baselines + baseline_dir - Optionally, specify a specific baseline dir, otherwise it will be computed from case config + + returns (SUCCESS, comments) """ rundir = case.get_value("RUNDIR") if baseline_dir is None: @@ -160,7 +202,7 @@ def generate_baseline(case, baseline_dir=None): num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '%s'\n" % model - hists = _get_all_hist_files(testcase, model, from_dir=rundir) + hists = _get_all_hist_files(testcase, model, rundir) num_gen += len(hists) for hist in hists: ext = get_extension(model, hist) diff --git a/utils/python/CIME/test_status.py b/utils/python/CIME/test_status.py index 7398ccf7b23..83e832d7254 100644 --- a/utils/python/CIME/test_status.py +++ b/utils/python/CIME/test_status.py @@ -62,7 +62,8 @@ def _test_helper2(file_contents, wait_for_run=False, check_throughput=False, che class TestStatus(object): - def __init__(self, test_dir=os.getcwd(), test_name=None): + def __init__(self, test_dir=None, test_name=None): + test_dir = os.getcwd() if test_dir is None else test_dir self._filename = os.path.join(test_dir, TEST_STATUS_FILENAME) self._phase_statuses = OrderedDict() # {name -> (status, comments)} self._test_name = test_name From 0c483c7cf61e7a135779da77adb3814a893326c8 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 11:56:27 -0600 Subject: [PATCH 08/20] Remove last cwd default args --- utils/python/CIME/hist_utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 33b814b88ec..daa003fe2d5 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -63,7 +63,9 @@ def move(case, suffix): return comments -def _compare_hists(case, suffix1="", suffix2="", from_dir1=os.getcwd(), from_dir2=os.getcwd()): +def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""): + if from_dir1 == from_dir2: + expect(suffix1 != suffix2, "Comparing files to themselves?") testcase = case.get_value("CASE") all_success = True @@ -111,7 +113,7 @@ def compare_test(case, suffix1, suffix2): """ rundir = case.get_value("RUNDIR") - return _compare_hists(case, suffix1, suffix2, rundir, rundir) + return _compare_hists(case, rundir, rundir, suffix1, suffix2) def cprnc(file1, file2, case, rundir): """ @@ -152,7 +154,7 @@ def compare_baseline(case, baseline_dir=None): if not os.path.isdir(bdir): return False, "ERROR %s does not exist" % bdir - return _compare_hists(case, from_dir1=rundir, from_dir2=basecmp_dir) + return _compare_hists(case, rundir, basecmp_dir) def get_extension(model, filepath): """ From cf4b7dfc16e32a3901b7e4c530803e20e54cc176 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Thu, 18 Aug 2016 13:12:16 -0600 Subject: [PATCH 09/20] fix issue in component_generate_baseline, get only most recent files --- scripts/Tools/component_generate_baseline | 2 +- utils/python/CIME/SystemTests/pea.py | 2 + utils/python/CIME/hist_utils.py | 45 ++++++++++++++++++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/scripts/Tools/component_generate_baseline b/scripts/Tools/component_generate_baseline index d352760effc..8fbc4bd2a25 100755 --- a/scripts/Tools/component_generate_baseline +++ b/scripts/Tools/component_generate_baseline @@ -39,7 +39,7 @@ OR CIME.utils.handle_standard_logging_options(args) - return args.caseroot + return args.caseroot, args.baseline_dir ############################################################################### def _main_func(description): diff --git a/utils/python/CIME/SystemTests/pea.py b/utils/python/CIME/SystemTests/pea.py index 61e7aa614b3..286be7da1b1 100644 --- a/utils/python/CIME/SystemTests/pea.py +++ b/utils/python/CIME/SystemTests/pea.py @@ -34,6 +34,8 @@ def build_phase(self, sharedlib_only=False, model_only=False): logging.warn("Starting bld for %s"%mpilib) self._case.set_value("MPILIB",mpilib) self._case.flush() + # We need to explicitly remove the Macros file for this test + os.remove("Macros") case_setup(self._case, reset=True) self.clean_build() self.build_indv(sharedlib_only=sharedlib_only, model_only=model_only) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index daa003fe2d5..9c7ce2a150b 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -5,6 +5,7 @@ from CIME.XML.standard_module_setup import * import logging, glob, os, shutil, re +logger = logging.getLogger(__name__) def _iter_model_file_substrs(case): @@ -32,6 +33,34 @@ def _get_all_hist_files(testcase, model, from_dir, suffix=""): test_hists.sort() return test_hists +def _get_latest_hist_files(testcase, model, from_dir, suffix=""): + test_hists = _get_all_hist_files(testcase, model, from_dir, suffix) + latest_files = {} + histlist = [] + date_match = re.compile(r"(\d\d\d\d)-(\d\d)-(\d\d)-(\d\d\d\d\d).nc") + for hist in test_hists: + ext = get_extension(model, hist) + if ext in latest_files: + s1 = date_match.search(hist) + if s1 is None: + latest_files[ext] = hist + continue + (yr,month,day,time) = s1.group(1,2,3,4) + s2 = date_match.search(latest_files[ext]) + (pyr,pmonth,pday,ptime) = s2.group(1,2,3,4) + if yr > pyr or (yr == pyr and month > pmonth) or \ + (yr == pyr and month == pmonth and day > pday) or \ + (yr == pyr and month == pmonth and day == pday and time > ptime): + latest_files[ext] = hist + logger.debug("ext %s hist %s %s"%(ext,hist,latest_files)) + else: + latest_files[ext] = hist + + for key in latest_files.keys(): + histlist.append(latest_files[key]) + return histlist + + def move(case, suffix): """ Change the suffix for the most recent batch of hist files in a case. This can @@ -49,7 +78,7 @@ def move(case, suffix): num_moved = 0 for model in _iter_model_file_substrs(case): comments += " Moving hist files for model '%s'\n" % model - test_hists = _get_all_hist_files(testcase, model, rundir) + test_hists = _get_latest_hist_files(testcase, model, rundir) num_moved += len(test_hists) for test_hist in test_hists: new_file = "%s.%s" % (test_hist, suffix) @@ -75,8 +104,8 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""): for model in _iter_model_file_substrs(case): comments += " comparing model '%s'\n" % model - hists1 = _get_all_hist_files(testcase, model, from_dir1, suffix1) - hists2 = _get_all_hist_files(testcase, model, from_dir2, suffix2) + hists1 = _get_latest_hist_files(testcase, model, from_dir1, suffix1) + hists2 = _get_latest_hist_files(testcase, model, from_dir2, suffix2) len_hist1 = len(hists1) len_hist2 = len(hists2) if len_hist1 == 0: @@ -152,7 +181,7 @@ def compare_baseline(case, baseline_dir=None): for bdir in dirs_to_check: if not os.path.isdir(bdir): - return False, "ERROR %s does not exist" % bdir + return False, "ERROR baseline directory %s not found" % bdir return _compare_hists(case, rundir, basecmp_dir) @@ -177,6 +206,9 @@ def get_extension(model, filepath): basename = os.path.basename(filepath) ext_regex = re.compile(r'.*%s.*[.](h.?)([.][^.]+)?[.]nc' % model) m = ext_regex.match(basename) + if m is None: + import pdb + pdb.set_trace() expect(m is not None, "Failed to get extension for file '%s'" % filepath) return m.groups()[0] @@ -204,7 +236,8 @@ def generate_baseline(case, baseline_dir=None): num_gen = 0 for model in _iter_model_file_substrs(case): comments += " generating for model '%s'\n" % model - hists = _get_all_hist_files(testcase, model, rundir) + hists = _get_latest_hist_files(testcase, model, rundir) + logger.debug("latest_files: %s"%hists) num_gen += len(hists) for hist in hists: ext = get_extension(model, hist) @@ -214,6 +247,6 @@ def generate_baseline(case, baseline_dir=None): os.remove(baseline) shutil.copy(hist, baseline) - comments += " generating baseline '%s'\n" % baseline + comments += " generating baseline '%s' from file %s\n" % (baseline, hist) return True, comments From 19a5b30ccd31d191092a316b9acfce165529c5eb Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 13:32:52 -0600 Subject: [PATCH 10/20] More fixes from review --- utils/python/CIME/hist_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 9c7ce2a150b..7aea732b6f7 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -181,7 +181,7 @@ def compare_baseline(case, baseline_dir=None): for bdir in dirs_to_check: if not os.path.isdir(bdir): - return False, "ERROR baseline directory %s not found" % bdir + return False, "ERROR baseline directory '%s' does not exist" % bdir return _compare_hists(case, rundir, basecmp_dir) @@ -249,4 +249,6 @@ def generate_baseline(case, baseline_dir=None): shutil.copy(hist, baseline) comments += " generating baseline '%s' from file %s\n" % (baseline, hist) + expect(num_gen > 0, "Could not generate any hist files for case '%s', something is seriously wrong" % testcase) + return True, comments From 7847e04771c0bcd39331b7c81907589e96689667 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 13:36:38 -0600 Subject: [PATCH 11/20] minor help string fix --- scripts/Tools/compare_test_results | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index 67c54ce053b..d31000b6753 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -3,7 +3,7 @@ """ Analyze results from a test root area, comparing non-BFB changes. Purpose is, instead of re-running tests -in generate mode, which is very slow, allow for very fast analsis of diffs. +in compare mode, which is very slow, allow for very fast analsis of diffs. You may need to load modules for cprnc to work. """ From 51afde00654f066e3dd9b6d51f3bacb75d65d181 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 14:55:02 -0600 Subject: [PATCH 12/20] Update hist infra to better-support user-chosen baseline_root --- scripts/Tools/bless_test_results | 10 +++++-- scripts/Tools/compare_test_results | 10 +++++-- utils/python/CIME/bless_test_results.py | 30 ++++++++----------- utils/python/CIME/compare_test_results.py | 8 ++--- .../python/tests/scripts_regression_tests.py | 2 +- 5 files changed, 31 insertions(+), 29 deletions(-) diff --git a/scripts/Tools/bless_test_results b/scripts/Tools/bless_test_results index fcfbdaa1c87..eb342ff54c5 100755 --- a/scripts/Tools/bless_test_results +++ b/scripts/Tools/bless_test_results @@ -48,6 +48,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ) default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root()) + default_baseline_root = _MACHINE.get_value("CCSM_BASELINE") default_compiler = _MACHINE.get_default_compiler() scratch_root = _MACHINE.get_value("CESMSCRATCHROOT") default_testroot = os.path.join(scratch_root) @@ -63,6 +64,9 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-b", "--baseline-name", default=default_baseline_name, help="Name of baselines to use, corresponds to branch used.") + parser.add_argument("--baseline-root", default=default_baseline_root, + help="Root of baselines.") + parser.add_argument("-c", "--compiler", default=default_compiler, help="Compiler of run you want to bless") @@ -90,7 +94,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter expect(not (args.namelists_only and args.hist_only), "Makes no sense to use --namelists-only and --hist-only simultaneously") - return args.baseline_name, args.test_root, args.compiler, args.test_id, args.namelists_only, args.hist_only, args.report_only, args.force, args.bless_tests + return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.namelists_only, args.hist_only, args.report_only, args.force, args.bless_tests ############################################################################### def _main_func(description): @@ -99,10 +103,10 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - baseline_name, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests = \ + baseline_name, baseline_root, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests = \ parse_command_line(sys.argv, description) - success = CIME.bless_test_results.bless_test_results(baseline_name, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests) + success = CIME.bless_test_results.bless_test_results(baseline_name, baseline_root, test_root, compiler, test_id, namelists_only, hist_only, report_only, force, bless_tests) sys.exit(0 if success else 1) ############################################################################### diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index d31000b6753..96038b3d0f2 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -43,6 +43,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter ) default_baseline_name = CIME.utils.get_current_branch(repo=CIME.utils.get_cime_root()) + default_baseline_root = _MACHINE.get_value("CCSM_BASELINE") default_compiler = _MACHINE.get_default_compiler() scratch_root = _MACHINE.get_value("CESMSCRATCHROOT") default_testroot = os.path.join(scratch_root) @@ -52,6 +53,9 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter parser.add_argument("-b", "--baseline-name", default=default_baseline_name, help="Name of baselines to use, corresponds to branch used.") + parser.add_argument("--baseline-root", default=default_baseline_root, + help="Root of baselines.") + parser.add_argument("-c", "--compiler", default=default_compiler, help="Compiler of run you want to compare") @@ -68,7 +72,7 @@ formatter_class=argparse.ArgumentDefaultsHelpFormatter CIME.utils.handle_standard_logging_options(args) - return args.baseline_name, args.test_root, args.compiler, args.test_id, args.compare_tests + return args.baseline_name, args.baseline_root, args.test_root, args.compiler, args.test_id, args.compare_tests ############################################################################### def _main_func(description): @@ -77,10 +81,10 @@ def _main_func(description): test_results = doctest.testmod(verbose=True) sys.exit(1 if test_results.failed > 0 else 0) - baseline_name, test_root, compiler, test_id, compare_tests = \ + baseline_name, baseline_root, test_root, compiler, test_id, compare_tests = \ parse_command_line(sys.argv, description) - success = CIME.compare_test_results.compare_test_results(baseline_name, test_root, compiler, test_id, compare_tests) + success = CIME.compare_test_results.compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id, compare_tests) sys.exit(0 if success else 1) ############################################################################### diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py index a4158291675..38d09fd0bc0 100644 --- a/utils/python/CIME/bless_test_results.py +++ b/utils/python/CIME/bless_test_results.py @@ -11,7 +11,7 @@ _MACHINE = Machines() ############################################################################### -def bless_namelists(test_name, report_only, force, baseline_name): +def bless_namelists(test_name, report_only, force, baseline_name, baseline_root): ############################################################################### # Be aware that restart test will overwrite the original namelist files # with versions of the files that should not be blessed. This forces us to @@ -21,21 +21,20 @@ def bless_namelists(test_name, report_only, force, baseline_name): print "Test '%s' had a namelist diff" % test_name if (not report_only and (force or raw_input("Update namelists (y/n)? ").upper() in ["Y", "YES"])): - stat, _, err = run_cmd("create_test -n -g %s -b %s" % (test_name, baseline_name)) + stat, _, err = run_cmd("create_test -n -g %s -b %s --baseline-root %s" % (test_name, baseline_name, baseline_root)) if stat != 0: return False, "Namelist regen failed: '%s'" % err else: return True, None ############################################################################### -def bless_history(test_name, testcase_dir_for_test, baseline_name, report_only, force): +def bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, report_only, force): ############################################################################### with Case(testcase_dir_for_test) as case: - baseline_full_dir = os.path.join(case.get_value("BASELINE_ROOT"), case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) + baseline_full_dir = os.path.join(baseline_root, case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) result, comments = compare_baseline(case, baseline_dir=baseline_full_dir) if result: - logging.warning("Test '%s' was marked as DIFF but compare_baseline did not find diff?" % test_name) - return False, "No diff found" + return True, None else: print comments if (not report_only and @@ -51,7 +50,7 @@ def bless_history(test_name, testcase_dir_for_test, baseline_name, report_only, return True, None ############################################################################### -def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelists_only=False, hist_only=False, report_only=False, force=False, bless_tests=None): +def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, namelists_only=False, hist_only=False, report_only=False, force=False, bless_tests=None): ############################################################################### test_id_glob = "*%s*%s*" % (compiler, baseline_name) if test_id is None else "*%s" % test_id test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) @@ -78,22 +77,17 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis # Compute hist status, False implies it diffed if (not namelists_only): run_result = ts.get_status(RUN_PHASE) - baseline_comp_result = ts.get_status("%s_baseline" % COMPARE_PHASE) if (run_result is None): broken_blesses.append((test_name, "no run phase")) logging.warning("Test '%s' did not make it to run phase" % test_name) hist_no_bless = True - elif (run_result == TEST_PASS_STATUS): - if (baseline_comp_result is None): - broken_blesses.append((test_name, "no history compare performed")) - logging.warning("Test '%s' had no history compare phase" % test_name) - hist_no_bless = True - else: - hist_no_bless = baseline_comp_result == TEST_PASS_STATUS - else: + elif (run_result != TEST_PASS_STATUS): broken_blesses.append((test_name, "test did not pass")) logging.warning("Test '%s' did not pass, not safe to bless" % test_name) hist_no_bless = True + else: + hist_no_bless = False + else: hist_no_bless = True @@ -124,13 +118,13 @@ def bless_test_results(baseline_name, test_root, compiler, test_id=None, namelis # Bless namelists if (not nl_no_bless): - success, reason = bless_namelists(test_name, report_only, force, baseline_name) + success, reason = bless_namelists(test_name, report_only, force, baseline_name, baseline_root) if not success: broken_blesses.append(test_name, reason) # Bless hist files if (not hist_no_bless): - success, reason = bless_history(test_name, testcase_dir_for_test, baseline_name, report_only, force) + success, reason = bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, report_only, force) if (not success): broken_blesses.append((test_name, reason)) diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py index 31957563dcc..45fba2937e9 100644 --- a/utils/python/CIME/compare_test_results.py +++ b/utils/python/CIME/compare_test_results.py @@ -10,10 +10,10 @@ _MACHINE = Machines() ############################################################################### -def compare_history(testcase_dir_for_test, baseline_name): +def compare_history(testcase_dir_for_test, baseline_name, baseline_root): ############################################################################### with Case(testcase_dir_for_test) as case: - baseline_full_dir = os.path.join(case.get_value("BASELINE_ROOT"), case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) + baseline_full_dir = os.path.join(baseline_root, case.get_value("COMPILER"), baseline_name, case.get_value("CASEBASEID")) result, comments = compare_baseline(case, baseline_dir=baseline_full_dir) if result: return True, None @@ -22,7 +22,7 @@ def compare_history(testcase_dir_for_test, baseline_name): return False, "Diff'd" ############################################################################### -def compare_test_results(baseline_name, test_root, compiler, test_id=None, compare_tests=None): +def compare_test_results(baseline_name, baseline_root, test_root, compiler, test_id=None, compare_tests=None): ############################################################################### test_id_glob = "*%s*%s*" % (compiler, baseline_name) if test_id is None else "*%s" % test_id test_status_files = glob.glob("%s/%s/%s" % (test_root, test_id_glob, TEST_STATUS_FILENAME)) @@ -73,7 +73,7 @@ def compare_test_results(baseline_name, test_root, compiler, test_id=None, compa # Compare hist files if (not hist_no_compare): - success, reason = compare_history(testcase_dir_for_test, baseline_name) + success, reason = compare_history(testcase_dir_for_test, baseline_name, baseline_root) if (not success): broken_compares.append((test_name, reason)) diff --git a/utils/python/tests/scripts_regression_tests.py b/utils/python/tests/scripts_regression_tests.py index e34f1874b47..870980cecfe 100755 --- a/utils/python/tests/scripts_regression_tests.py +++ b/utils/python/tests/scripts_regression_tests.py @@ -750,7 +750,7 @@ def test_bless_test_results(self): expected_pattern = re.compile(r'COMPARE FAILED FOR TEST: %s[^\s]* reason Diff' % self._test_name) the_match = expected_pattern.search(out) self.assertNotEqual(the_match, None, - msg="Failed to display failed test in output:\n%s" % out) + msg="Cmd '%s' failed to display failed test in output:\n%s" % (cpr_cmd, out)) # Bless run_cmd_no_fail("%s/bless_test_results --hist-only --force -b %s -t %s" % (TOOLS_DIR, self._baseline_name, test_id)) From 51e181b13d7cfbf4da39d315367fa0eae8c27eff Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 15:02:16 -0600 Subject: [PATCH 13/20] Remove unneeded global --- utils/python/CIME/bless_test_results.py | 3 --- utils/python/CIME/compare_test_results.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py index 38d09fd0bc0..8e9ba28fb9a 100644 --- a/utils/python/CIME/bless_test_results.py +++ b/utils/python/CIME/bless_test_results.py @@ -1,15 +1,12 @@ import CIME.compare_namelists, CIME.simple_compare from CIME.test_scheduler import NAMELIST_PHASE from CIME.utils import run_cmd, expect -from CIME.XML.machines import Machines from CIME.test_status import * from CIME.hist_utils import generate_baseline, compare_baseline from CIME.case import Case import os, glob, time -_MACHINE = Machines() - ############################################################################### def bless_namelists(test_name, report_only, force, baseline_name, baseline_root): ############################################################################### diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py index 45fba2937e9..33b84139f4c 100644 --- a/utils/python/CIME/compare_test_results.py +++ b/utils/python/CIME/compare_test_results.py @@ -1,14 +1,11 @@ import CIME.compare_namelists, CIME.simple_compare from CIME.utils import expect -from CIME.XML.machines import Machines from CIME.test_status import * from CIME.hist_utils import compare_baseline from CIME.case import Case import os, glob -_MACHINE = Machines() - ############################################################################### def compare_history(testcase_dir_for_test, baseline_name, baseline_root): ############################################################################### From df31225c20209a60a4b9c3d20da1fa8db24f942b Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 15:37:33 -0600 Subject: [PATCH 14/20] Make a very obvious simplification to code --- utils/python/CIME/bless_test_results.py | 19 +++---------------- utils/python/CIME/compare_test_results.py | 19 +++---------------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/utils/python/CIME/bless_test_results.py b/utils/python/CIME/bless_test_results.py index 8e9ba28fb9a..566c9a5a76e 100644 --- a/utils/python/CIME/bless_test_results.py +++ b/utils/python/CIME/bless_test_results.py @@ -55,7 +55,8 @@ def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_i broken_blesses = [] for test_status_file in test_status_files: - ts = TestStatus(test_dir=os.path.dirname(test_status_file)) + test_dir = os.path.dirname(test_status_file) + ts = TestStatus(test_dir=test_dir) test_name = ts.get_name() if (bless_tests in [[], None] or CIME.utils.match_any(test_name, bless_tests)): overall_result = ts.get_overall_test_status() @@ -99,20 +100,6 @@ def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_i if not force: time.sleep(2) - # Get testcase dir for this test - if (test_id is None): - # The full name already contains the compiler, so we just need to glob for the branch name - globs = glob.glob("%s/%s*%s*" % (test_root, test_name, baseline_name)) - else: - globs = glob.glob("%s/%s%s" % (test_root, test_name, test_id_glob)) - - if (len(globs) != 1): - logging.warning("Expected exactly one match for testcase area for test '%s', found '%s'" % (test_name, globs)) - broken_blesses.append((test_name, "multiple matching testcase dirs")) - continue - - testcase_dir_for_test = globs[0] - # Bless namelists if (not nl_no_bless): success, reason = bless_namelists(test_name, report_only, force, baseline_name, baseline_root) @@ -121,7 +108,7 @@ def bless_test_results(baseline_name, baseline_root, test_root, compiler, test_i # Bless hist files if (not hist_no_bless): - success, reason = bless_history(test_name, testcase_dir_for_test, baseline_name, baseline_root, report_only, force) + success, reason = bless_history(test_name, test_dir, baseline_name, baseline_root, report_only, force) if (not success): broken_blesses.append((test_name, reason)) diff --git a/utils/python/CIME/compare_test_results.py b/utils/python/CIME/compare_test_results.py index 33b84139f4c..8731b30de2b 100644 --- a/utils/python/CIME/compare_test_results.py +++ b/utils/python/CIME/compare_test_results.py @@ -27,7 +27,8 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test broken_compares = [] for test_status_file in test_status_files: - ts = TestStatus(test_dir=os.path.dirname(test_status_file)) + test_dir = os.path.dirname(test_status_file) + ts = TestStatus(test_dir=test_dir) test_name = ts.get_name() if (compare_tests in [[], None] or CIME.utils.match_any(test_name, compare_tests)): overall_result = ts.get_overall_test_status() @@ -54,23 +55,9 @@ def compare_test_results(baseline_name, baseline_root, test_root, compiler, test logging.info("Comparing results for test: %s, most recent result: %s" % (test_name, overall_result)) logging.info("###############################################################################") - # Get testcase dir for this test - if (test_id is None): - # The full name already contains the compiler, so we just need to glob for the branch name - globs = glob.glob("%s/%s*%s*" % (test_root, test_name, baseline_name)) - else: - globs = glob.glob("%s/%s%s" % (test_root, test_name, test_id_glob)) - - if (len(globs) != 1): - logging.warning("Expected exactly one match for testcase area for test '%s', found '%s'" % (test_name, globs)) - broken_compares.append((test_name, "multiple matching testcase dirs")) - continue - - testcase_dir_for_test = globs[0] - # Compare hist files if (not hist_no_compare): - success, reason = compare_history(testcase_dir_for_test, baseline_name, baseline_root) + success, reason = compare_history(test_dir, baseline_name, baseline_root) if (not success): broken_compares.append((test_name, reason)) From 9fac682d9dd4ddd2bc15d09abf671454054a94b9 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Thu, 18 Aug 2016 16:29:27 -0600 Subject: [PATCH 15/20] Get rid of pdb trace that I believe was mistakenly left in --- utils/python/CIME/hist_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 7aea732b6f7..372e372a5ab 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -206,9 +206,6 @@ def get_extension(model, filepath): basename = os.path.basename(filepath) ext_regex = re.compile(r'.*%s.*[.](h.?)([.][^.]+)?[.]nc' % model) m = ext_regex.match(basename) - if m is None: - import pdb - pdb.set_trace() expect(m is not None, "Failed to get extension for file '%s'" % filepath) return m.groups()[0] From 4e5facfadf3072bf69c155ee3583178458744771 Mon Sep 17 00:00:00 2001 From: Bill Sacks Date: Thu, 18 Aug 2016 19:34:18 -0600 Subject: [PATCH 16/20] Add usage example for typical CESM workflow --- scripts/Tools/compare_test_results | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index 96038b3d0f2..ce8afb19ec2 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -35,7 +35,9 @@ OR > %s -n foo bar \033[1;32m# From most recent run of jenkins, compare history changes for next \033[0m > %s -r /home/jenkins/acme/scratch/jenkins -b next -""" % ((os.path.basename(args[0]), ) * 6), + \033[1;32m# For typical CESM workflow, where baselines are named with tags \033[0m + > %s -t TESTID -b BASELINE_TAG +""" % ((os.path.basename(args[0]), ) * 7), description=description, From 96c3c18bb5126baec2d0b6b443ac8c66a9c8c25a Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 19 Aug 2016 08:32:45 -0600 Subject: [PATCH 17/20] correct location of debug log in help message, store baselines with original filename --- utils/python/CIME/hist_utils.py | 3 +-- utils/python/CIME/utils.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 372e372a5ab..91c5990be6f 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -238,8 +238,7 @@ def generate_baseline(case, baseline_dir=None): num_gen += len(hists) for hist in hists: ext = get_extension(model, hist) - basename = "%s.%s.nc" % (model, ext) - baseline = os.path.join(basegen_dir, basename) + baseline = os.path.join(basegen_dir, os.path.basename(hist)) if os.path.exists(baseline): os.remove(baseline) diff --git a/utils/python/CIME/utils.py b/utils/python/CIME/utils.py index 947db7276c4..fbf7a675eb1 100644 --- a/utils/python/CIME/utils.py +++ b/utils/python/CIME/utils.py @@ -541,8 +541,10 @@ def get_project(machobj=None): return None def setup_standard_logging_options(parser): + helpfile = "%s.log"%sys.argv[0] + helpfile = os.path.join(os.getcwd(),os.path.basename(helpfile)) parser.add_argument("-d", "--debug", action="store_true", - help="Print debug information (very verbose) to file %s.log" % sys.argv[0]) + help="Print debug information (very verbose) to file %s" % helpfile) parser.add_argument("-v", "--verbose", action="store_true", help="Add additional context (time and file) to log messages") parser.add_argument("-s", "--silent", action="store_true", From f6ea4a1c98bab31f9fb83773e06ba268e687e0b6 Mon Sep 17 00:00:00 2001 From: Jim Edwards Date: Fri, 19 Aug 2016 09:49:48 -0600 Subject: [PATCH 18/20] improved reporting of baseline file count mismatch --- utils/python/CIME/hist_utils.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 91c5990be6f..3e08997f453 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -111,20 +111,29 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""): if len_hist1 == 0: comments += " no hist files found for model %s\n"%model continue - if len_hist1 != len_hist2: + if len_hist1 > len_hist2: comments += " num hists does not match %d != %d\n" % (len_hist1, len_hist2) all_success = False - continue + hists2 += ['MISSING'] * (len_hist1 - len_hist2) + if len_hist1 < len_hist2: + comments += " num hists does not match %d != %d\n" % (len_hist1, len_hist2) + all_success = False + hists1 += ['MISSING'] * (len_hist2 - len_hist1) num_compared += len(hists1) for hist1, hist2 in zip(hists1, hists2): - success, cprnc_comments = cprnc(hist1, hist2, case, from_dir1) - if success: - comments += " %s matched %s\n" % (hist1, hist2) + if hist1 == "MISSING": + comments += " No match for file %s found in %s\n"%(hist2, from_dir1) + elif hist2 == "MISSING": + comments += " No match for file %s found in %s\n"%(hist1, from_dir2) else: - comments += " %s did NOT match %s\n" % (hist1, hist2) - comments += cprnc_comments + "\n" - all_success = False + success, cprnc_comments = cprnc(hist1, hist2, case, from_dir1) + if success: + comments += " %s matched %s\n" % (hist1, hist2) + else: + comments += " %s did NOT match %s\n" % (hist1, hist2) + comments += cprnc_comments + "\n" + all_success = False expect(num_compared > 0, "Did not compare any hist files for suffix1='%s' suffix2='%s', dir1='%s', dir2='%s'\nComments=%s" % (suffix1, suffix2, from_dir1, from_dir2, comments)) From 6fb02205bfd739d184022e654f762e00335ebb0f Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 19 Aug 2016 10:52:34 -0600 Subject: [PATCH 19/20] Fix user docs for compare_test_results --- scripts/Tools/compare_test_results | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/Tools/compare_test_results b/scripts/Tools/compare_test_results index ce8afb19ec2..1e5f721bf9f 100755 --- a/scripts/Tools/compare_test_results +++ b/scripts/Tools/compare_test_results @@ -22,7 +22,7 @@ _MACHINE = Machines() def parse_command_line(args, description): ############################################################################### parser = argparse.ArgumentParser( -usage="""\n%s [-n] [-r ] [-b ] [-c ] [ ...] [--verbose] +usage="""\n%s [-r ] [-b -c ] [-t ] [ ...] [--verbose] OR %s --help OR @@ -32,12 +32,14 @@ OR \033[1;32m# From most recent run, compare all changes \033[0m > %s \033[1;32m# From most recent run, compare only changes for test foo and bar only \033[0m - > %s -n foo bar + > %s foo bar + \033[1;32m# For an old run where you know test-id, compare only changes for test foo and bar only \033[0m + > %s foo bar -t mytestid \033[1;32m# From most recent run of jenkins, compare history changes for next \033[0m > %s -r /home/jenkins/acme/scratch/jenkins -b next \033[1;32m# For typical CESM workflow, where baselines are named with tags \033[0m > %s -t TESTID -b BASELINE_TAG -""" % ((os.path.basename(args[0]), ) * 7), +""" % ((os.path.basename(args[0]), ) * 8), description=description, From 164cfd912e49fb133e2cbb35579c262e1edf4bf5 Mon Sep 17 00:00:00 2001 From: James Foucar Date: Fri, 19 Aug 2016 15:17:18 -0600 Subject: [PATCH 20/20] Make comparison matchups more robust --- .../CIME/SystemTests/system_tests_common.py | 5 +- utils/python/CIME/hist_utils.py | 87 +++++++++++++------ 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/utils/python/CIME/SystemTests/system_tests_common.py b/utils/python/CIME/SystemTests/system_tests_common.py index 14ef1b8ce2c..099c68a880c 100644 --- a/utils/python/CIME/SystemTests/system_tests_common.py +++ b/utils/python/CIME/SystemTests/system_tests_common.py @@ -1,7 +1,6 @@ """ Base class for CIME system tests """ -import shutil, glob, gzip, time from CIME.XML.standard_module_setup import * from CIME.XML.env_run import EnvRun from CIME.utils import append_status @@ -13,6 +12,8 @@ import CIME.build as build +import shutil, glob, gzip, time, traceback + logger = logging.getLogger(__name__) class SystemTestsCommon(object): @@ -72,6 +73,7 @@ def build(self, sharedlib_only=False, model_only=False): except: success = False logger.warning("Exception during build:\n%s" % (sys.exc_info()[1])) + logger.warning(traceback.format_exc()) time_taken = time.time() - start_time with self._test_status: @@ -128,6 +130,7 @@ def run(self): except: success = False logger.warning("Exception during run:\n%s" % (sys.exc_info()[1])) + logger.warning(traceback.format_exc()) # Always try to report, should NOT throw an exception self.report() diff --git a/utils/python/CIME/hist_utils.py b/utils/python/CIME/hist_utils.py index 3e08997f453..395af87fa95 100644 --- a/utils/python/CIME/hist_utils.py +++ b/utils/python/CIME/hist_utils.py @@ -8,7 +8,6 @@ logger = logging.getLogger(__name__) def _iter_model_file_substrs(case): - models = case.get_compset_components() models.append('cpl') for model in models: @@ -29,6 +28,8 @@ def _get_all_hist_files(testcase, model, from_dir, suffix=""): if suffix == "": test_hists.extend(glob.glob("%s/%s.h.nc" % (from_dir, model))) test_hists.extend(glob.glob("%s/%s.h?.nc" % (from_dir, model))) + test_hists.extend(glob.glob("%s/%s.h.*.nc" % (from_dir, model))) + test_hists.extend(glob.glob("%s/%s.h?.*.nc" % (from_dir, model))) test_hists.sort() return test_hists @@ -92,6 +93,42 @@ def move(case, suffix): return comments +def _hists_match(model, hists1, hists2, suffix1="", suffix2=""): + """ + return (num in set 1 but not 2 , num in set 2 but not 1, matchups) + + >>> hists1 = ['FOO.G.cpl.h1.nc', 'FOO.G.cpl.h2.nc', 'FOO.G.cpl.h3.nc'] + >>> hists2 = ['cpl.h2.nc', 'cpl.h3.nc', 'cpl.h4.nc'] + >>> _hists_match('cpl', hists1, hists2) + (['FOO.G.cpl.h1.nc'], ['cpl.h4.nc'], [('FOO.G.cpl.h2.nc', 'cpl.h2.nc'), ('FOO.G.cpl.h3.nc', 'cpl.h3.nc')]) + >>> hists1 = ['FOO.G.cpl.h1.nc.SUF1', 'FOO.G.cpl.h2.nc.SUF1', 'FOO.G.cpl.h3.nc.SUF1'] + >>> hists2 = ['cpl.h2.nc.SUF2', 'cpl.h3.nc.SUF2', 'cpl.h4.nc.SUF2'] + >>> _hists_match('cpl', hists1, hists2, 'SUF1', 'SUF2') + (['FOO.G.cpl.h1.nc.SUF1'], ['cpl.h4.nc.SUF2'], [('FOO.G.cpl.h2.nc.SUF1', 'cpl.h2.nc.SUF2'), ('FOO.G.cpl.h3.nc.SUF1', 'cpl.h3.nc.SUF2')]) + """ + normalized1, normalized2 = [], [] + for hists, suffix, normalized in [(hists1, suffix1, normalized1), (hists2, suffix2, normalized2)]: + for hist in hists: + normalized_name = hist[hist.rfind(model):] + if suffix1 != "": + expect(normalized_name.endswith(suffix), "How did '%s' hot have suffix '%s'" % (hist, suffix)) + normalized_name = normalized_name[:len(normalized_name) - len(suffix)] + + normalized.append(normalized_name) + + set_of_1_not_2 = set(normalized1) - set(normalized2) + set_of_2_not_1 = set(normalized2) - set(normalized1) + + one_not_two = sorted([hists1[normalized1.index(item)] for item in set_of_1_not_2]) + two_not_one = sorted([hists2[normalized2.index(item)] for item in set_of_2_not_1]) + + both = set(normalized1) & set(normalized2) + match_ups = sorted([ (hists1[normalized1.index(item)], hists2[normalized2.index(item)]) for item in both]) + expect(len(match_ups) + len(set_of_1_not_2) == len(hists1), "Programming error1") + expect(len(match_ups) + len(set_of_2_not_1) == len(hists2), "Programming error2") + + return one_not_two, two_not_one, match_ups + def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""): if from_dir1 == from_dir2: expect(suffix1 != suffix2, "Comparing files to themselves?") @@ -106,34 +143,28 @@ def _compare_hists(case, from_dir1, from_dir2, suffix1="", suffix2=""): comments += " comparing model '%s'\n" % model hists1 = _get_latest_hist_files(testcase, model, from_dir1, suffix1) hists2 = _get_latest_hist_files(testcase, model, from_dir2, suffix2) - len_hist1 = len(hists1) - len_hist2 = len(hists2) - if len_hist1 == 0: - comments += " no hist files found for model %s\n"%model + + if len(hists1) == 0 and len(hists2) == 0: + comments += " no hist files found for model %s\n" % model continue - if len_hist1 > len_hist2: - comments += " num hists does not match %d != %d\n" % (len_hist1, len_hist2) + + one_not_two, two_not_one, match_ups = _hists_match(model, hists1, hists2, suffix1, suffix2) + for item in one_not_two: + comments += " File '%s' had no counterpart in '%s' with suffix '%s'\n" % (item, from_dir2, suffix2) all_success = False - hists2 += ['MISSING'] * (len_hist1 - len_hist2) - if len_hist1 < len_hist2: - comments += " num hists does not match %d != %d\n" % (len_hist1, len_hist2) + for item in two_not_one: + comments += " File '%s' had no counterpart in '%s' with suffix '%s'\n" % (item, from_dir1, suffix1) all_success = False - hists1 += ['MISSING'] * (len_hist2 - len_hist1) - - num_compared += len(hists1) - for hist1, hist2 in zip(hists1, hists2): - if hist1 == "MISSING": - comments += " No match for file %s found in %s\n"%(hist2, from_dir1) - elif hist2 == "MISSING": - comments += " No match for file %s found in %s\n"%(hist1, from_dir2) + + num_compared += len(match_ups) + for hist1, hist2 in match_ups: + success, cprnc_comments = cprnc(hist1, hist2, case, from_dir1) + if success: + comments += " %s matched %s\n" % (hist1, hist2) else: - success, cprnc_comments = cprnc(hist1, hist2, case, from_dir1) - if success: - comments += " %s matched %s\n" % (hist1, hist2) - else: - comments += " %s did NOT match %s\n" % (hist1, hist2) - comments += cprnc_comments + "\n" - all_success = False + comments += " %s did NOT match %s\n" % (hist1, hist2) + comments += cprnc_comments + "\n" + all_success = False expect(num_compared > 0, "Did not compare any hist files for suffix1='%s' suffix2='%s', dir1='%s', dir2='%s'\nComments=%s" % (suffix1, suffix2, from_dir1, from_dir2, comments)) @@ -243,11 +274,11 @@ def generate_baseline(case, baseline_dir=None): for model in _iter_model_file_substrs(case): comments += " generating for model '%s'\n" % model hists = _get_latest_hist_files(testcase, model, rundir) - logger.debug("latest_files: %s"%hists) + logger.debug("latest_files: %s" % hists) num_gen += len(hists) for hist in hists: - ext = get_extension(model, hist) - baseline = os.path.join(basegen_dir, os.path.basename(hist)) + basename = hist[hist.rfind(model):] + baseline = os.path.join(basegen_dir, basename) if os.path.exists(baseline): os.remove(baseline)