From 2c867aadf0a3bab77ab2c1f6639a475b504bf23d Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 14 Feb 2019 19:32:36 +0000 Subject: [PATCH 1/2] add tdir cice.setup option to define test directory --- cice.setup | 47 +++++++++++++-------- doc/source/user_guide/ug_running.rst | 2 +- doc/source/user_guide/ug_testing.rst | 62 +++++++++++++++++++--------- 3 files changed, 72 insertions(+), 39 deletions(-) diff --git a/cice.setup b/cice.setup index a8f8d8e90..a8226e69e 100755 --- a/cice.setup +++ b/cice.setup @@ -19,6 +19,7 @@ set test = ${spval} set grid = gx3 set pesx = 4x1 set sets = "" +set tdir = ${spval} set bdir = ${spval} set testid = ${spval} set testsuite = ${spval} @@ -95,9 +96,10 @@ DESCRIPTION --test : test, test name (not with --case or --suite) --suite : test suite, pre-defined set or sets of tests, comma separated (not with --case or --test) - --bdir : top baseline directory, default ICE_MACHINE_BASELINE - --bgen : baselines directory where output from current tests are copied - --bcmp : baselines directory where output from current tests are compared + --tdir : directory name where tests will be located + --bdir : baseline directory for regression testing, default ICE_MACHINE_BASELINE + --bgen : directory name where output from current tests are copied + --bcmp : directory name where output from current tests are compared --testid : test ID, user-defined id for testing (REQUIRED with --test or --suite) --diff : generate comparison against another case --report : automatically post results when tests are complete @@ -234,6 +236,8 @@ while (1) else if ("$option" == "--test") then set test = $argv[1] set dotest = 1 + else if ("$option" == "--tdir") then + set tdir = $argv[1] else if ("$option" == "--grid" || "$option" == "-g") then set grid = $argv[1] else if ("$option" == "--queue") then @@ -314,6 +318,9 @@ if ( ${dosuite} == 1 ) then set tsdir = "testsuite.${testid}" set tsfile = "testsuite.${testid}.${sdate}-${stime}.list" endif +if ( ${tdir} != ${spval} ) then + set tsdir = ${tdir} +endif if (-e $tsfile) then echo "${0}: ERROR in tsfile, this should never happen" exit -1 @@ -348,20 +355,24 @@ else endif end - if (-e ./${tsdir}) then + if (-e ${tsdir}) then echo "${0}: ERROR, ${tsdir} already exists" exit -1 endif - mkdir ./${tsdir} - cp -f ${ICE_SCRIPTS}/tests/report_results.csh ./${tsdir} - cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ./${tsdir} + mkdir -p ${tsdir} + if ($status != 0) then + echo "${0}: ERROR, mkdir ${tsdir} aborted" + exit -1 + endif + cp -f ${ICE_SCRIPTS}/tests/report_results.csh ${tsdir} + cp -f ${ICE_SCRIPTS}/tests/timeseries.csh ${tsdir} if ($report == 1) then - cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ./${tsdir} + cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ${tsdir} endif -cat >! ./${tsdir}/suite.run << EOF0 +cat >! ${tsdir}/suite.run << EOF0 #!/bin/csh -f EOF0 @@ -375,7 +386,7 @@ EOF0 set vers = ${ICE_VERSION} set shhash = `echo ${hash} | cut -c 1-10` -cat >! ./${tsdir}/results.csh << EOF0 +cat >! ${tsdir}/results.csh << EOF0 #!/bin/csh -f rm -f results.log echo "#------- " >> results.log @@ -393,8 +404,8 @@ echo "#vers = ${vers}" >> results.log echo "#------- " >> results.log EOF0 - chmod +x ./${tsdir}/suite.run - chmod +x ./${tsdir}/results.csh + chmod +x ${tsdir}/suite.run + chmod +x ${tsdir}/results.csh endif @@ -469,9 +480,9 @@ foreach compiler ( $ncompilers ) set testname_noid = "${machcomp}_${test}_${grid}_${pesx}${soptions}" set testname_base = "${machcomp}_${test}_${grid}_${pesx}${soptions}.${testid}" if (${dosuite} == 1) then - set testname = "${tsdir}/$testname_base" + set testname = "${tsdir}/${testname_base}" else - set testname = "$testname_base" + set testname = "${testname_base}" endif set case = ${testname} endif @@ -809,11 +820,11 @@ EOF2 cd ${ICE_SANDBOX} # Write build and run commands to suite.run -cat >> ./${tsdir}/results.csh << EOF -cat $testname_base/test_output >> results.log +cat >> ${tsdir}/results.csh << EOF +cat ${testname_base}/test_output >> results.log EOF -cat >> ./${tsdir}/suite.run << EOF +cat >> ${tsdir}/suite.run << EOF echo "-------test--------------" echo "${testname_base}" cd ${testname_base} @@ -845,7 +856,7 @@ end if ( ${dosuite} == 1 ) then # Add code to results.csh to count the number of failures -cat >> ./${tsdir}/results.csh << EOF +cat >> ${tsdir}/results.csh << EOF cat ./results.log set pends = \`cat ./results.log | grep PEND | wc -l\` set failures = \`cat ./results.log | grep FAIL | wc -l\` diff --git a/doc/source/user_guide/ug_running.rst b/doc/source/user_guide/ug_running.rst index cd96e9d69..ad3f5630d 100644 --- a/doc/source/user_guide/ug_running.rst +++ b/doc/source/user_guide/ug_running.rst @@ -36,7 +36,7 @@ There are three usage modes, All modes will require use of ``--mach`` or ``-m`` to specify the machine and case and test modes can use ``--set`` or ``-s`` to define specific options. ``--test`` and ``--suite`` will require ``--testid`` to be set -and both of the test modes can use ``--bdir``, ``--bgen``, ``--bcmp``, and ``--diff`` to generate (save) results and compare results with prior results. +and both of the test modes can use ``--bdir``, ``--bgen``, ``--bcmp``, and ``--diff`` to generate (save) results and compare results with prior results as well as ``--tdir`` to specify the location of the test directory. Testing will be described in greater detail in the :ref:`testing` section. Again, ``cice.setup --help`` will show the latest usage information including diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index aa6804701..3ac099f9e 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -47,6 +47,9 @@ For individual tests, the following command line options can be set ``--testid`` ID specifies the testid. This is required for every use of ``--test`` and ``--suite``. This is a user defined string that will allow each test to have a unique case and run directory name. This is also required. +``--tdir`` PATH + specifies the test directory. Testcases will be created in this directory. (default is .) + ``--mach`` MACHINE (see :ref:`case_options`) ``--env`` ENVIRONMENT1 (see :ref:`case_options`) @@ -295,6 +298,12 @@ results.csh script in the [suite_name].[testid]:: To report the test results, as is required for Pull Requests to be accepted into the master the CICE Consortium code see :ref:`testreporting`. +If using the ``--tdir`` option, that directory must not exist before the script is run. The tdir directory will be +created by the script and it will be populated by all tests as well as scripts that support the +test suite:: + + ./cice.setup --suite base_suite --mach wolf --env gnu --testid myid --tdir /scratch/$user/testsuite.myid + Multiple suites are supported on the command line as comma separated arguments:: ./cice.setup --suite base_suite,decomp_suite --mach wolf --env gnu --testid myid @@ -353,6 +362,9 @@ following options are valid for suites, ``--acct`` ACCOUNT optional +``--tdir`` PATH + optional + ``--testid`` ID required @@ -380,93 +392,103 @@ Test Suite Examples :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a - cd base_suite.v01a + cd testsuite.v01a + #wait for runs to complete + ./results.csh + + 2) **Basic test suite with user defined test directory** + + Specify suite, mach, env, testid, tdir. + :: + + ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --tdir /scratch/$user/ts.v01a + cd /scratch/$user/ts.v01a #wait for runs to complete ./results.csh - 2) **Basic test suite on multiple environments** + 3) **Basic test suite on multiple environments** Specify multiple envs. :: ./cice.setup --suite base_suite --mach conrad --env cray,pgi,intel,gnu --testid v01a - cd base_suite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh Each env can be run as a separate invokation of `cice.setup` but if that approach is taken, it is recommended that different testids be used. - 3) **Basic test suite with generate option defined** + 4) **Basic test suite with generate option defined** Add ``--set`` :: ./cice.setup --suite base_suite --mach conrad --env gnu --testid v01b --set diag1 - cd base_suite.v01b + cd testsuite.v01b #wait for runs to complete ./results.csh If there are conflicts between the ``--set`` options in the suite and on the command line, the suite will take precedent. - 4) **Multiple test suites from a single command line** + 5) **Multiple test suites from a single command line** Add comma delimited list of suites :: ./cice.setup --suite base_suite,decomp_suite --mach conrad --env gnu --testid v01c - cd base_suite.v01c + cd testsuite.v01c #wait for runs to complete ./results.csh If there are redundant tests in multiple suites, the scripts will understand that and only create one test. - 5) **Basic test suite, store baselines in user defined name** + 6) **Basic test suite, store baselines in user defined name** Add ``--bgen`` :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a - cd base_suite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh This will store the results in the default [bdir] directory under the subdirectory cice.v01a. - 6) **Basic test suite, store baselines in user defined top level directory** + 7) **Basic test suite, store baselines in user defined top level directory** Add ``--bgen`` and ``--bdir`` :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen cice.v01a --bdir /tmp/user/CICE_BASELINES - cd base_suite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh This will store the results in /tmp/user/CICE_BASELINES/cice.v01a. - 7) **Basic test suite, store baselines in auto-generated directory** + 8) **Basic test suite, store baselines in auto-generated directory** Add ``--bgen default`` :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a --bgen default - cd base_suite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh This will store the results in the default [bdir] directory under a directory name generated by the script that includes the hash and date. - 8) **Basic test suite, compare to prior baselines** + 9) **Basic test suite, compare to prior baselines** Add ``--bcmp`` :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v02a --bcmp cice.v01a - cd base_suite.v02a + cd testsuite.v02a #wait for runs to complete ./results.csh @@ -477,7 +499,7 @@ Test Suite Examples the CICE Consortium master code. You can use other regression options as well. (``--bdir`` and ``--bgen``) - 9) **Basic test suite, use of default string in regression testing** + 10) **Basic test suite, use of default string in regression testing** default is a special argument to ``--bgen`` and ``--bcmp``. When used, the scripts will automate generation of the directories. In the case of ``--bgen``, @@ -504,7 +526,7 @@ Test Suite Examples When this is invoked, a new set of baselines will be generated and compared to the prior results each time without having to change the arguments. - 10) **Create and test a custom suite** + 11) **Create and test a custom suite** Create your own input text file consisting of 5 columns of data, - Test @@ -526,7 +548,7 @@ Test Suite Examples :: ./cice.setup --suite mysuite --mach conrad --env cray --testid v01a --bgen default - cd mysuite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh @@ -557,7 +579,7 @@ To post results, once a test suite is complete, run ``results.csh`` and :: ./cice.setup --suite base_suite --mach conrad --env cray --testid v01a - cd base_suite.v01a + cd testsuite.v01a #wait for runs to complete ./results.csh ./report_results.csh @@ -926,7 +948,7 @@ $ ./cice.setup -m conrad -e intel --suite base_suite --testid t00 Wait for suite to finish then go to the directory. :: -$ cd base_suite.t00 +$ cd testsuite.t00 Run the timeseries script on the desired case. :: From 8c6111d8614f8bd93c642aece222d5a45d8a0ea7 Mon Sep 17 00:00:00 2001 From: apcraig Date: Thu, 14 Feb 2019 20:45:59 +0000 Subject: [PATCH 2/2] update documentation --- cice.setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cice.setup b/cice.setup index a8226e69e..6952a2de5 100755 --- a/cice.setup +++ b/cice.setup @@ -74,10 +74,11 @@ SYNOPSIS --test TEST -m MACH --testid ID [-e ENV][-p MxN][-g GRID][-s SET1,SET2][--acct ACCT] [--diff TESTNAME][--bdir DIR][--bgen DIR][--bcmp DIR] + [--tdir PATH] --suite SUITE[,SUITE2] -m MACH --testid ID [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR] - [--bcmp DIR][--report] + [--bcmp DIR][--tdir PATH][--report] DESCRIPTION --help, -h : help