Testing suite

README.test

@@ -0,0 +1,79 @@
+CICE version 6 testing
+  June 2017
+
+This README file documents how to use the testing suite developed for the
+CICE Consortium sea ice model.  The tests currently supported are:
+  1. "Smoke" test  - Ensures that the model compiles successfully and runs
+                     to completion for a 1 day simulation.  Validation is
+                     determined by performing a binary comparison on restart
+                     files to a baseline dataset.  This test is created by
+                     passing 'smoke' to the '-t' argument.
+  2. 10-day test   - Ensures that the model compiles successfully and runs
+                     to completion for a 10 day simulation.  Validation is
+                     determined by performing a binary comparison on restart
+                     files to a baseline dataset.  This test is created by 
+                     passing '10day' to the '-t' argument.
+  3. Exact restart - Ensures that the output from a cold-start simulation
+                     and a restart simulation provide bit-for-bit identical
+                     results.  No baseline dataset needs to be created for 
+                     this test, as the baseline and test datasets are computed
+                     at the same time.  This test is created by passing 'restart'
+                     to the '-t' argument.
+  4. Annual test   - Ensures that the model compiles successfully and runs
+                     to completion for an annual simulation.  Validation is
+                     determined by performing a binary comparison on restart
+                     files to a baseline dataset.  This test is created by 
+                     passing 'annual' to the '-t' argument.
+
+Please run './create.case -h' for additional details.
+
+--- To generate a baseline dataset for a test case ---
+
+Quickstart (example):
+
+./create.case -t smoke -m conrad -b
+cd smoke_gx3_conrad_4x1
+./cice.build
+./cice.submit
+# After job finishes, check output
+cat test_output
+
+
+--- To run a test case and compare to a baseline dataset ---
+
+Quickstart (example):
+
+./create.case -t smoke -m conrad
+cd smoke_gx3_conrad_4x1.t00
+./cice.build
+./cice.submit
+# After job finishes, check output
+cat test_output
+
+
+--- To run a regression test (compare cicev6.0.1 output to cicev6.0.0 output) ---
+
+Quickstart (example):
+
+./create.case -t smoke -m conrad -b -bg cicev6.0.1 -bc cicev6.0.0
+cd smoke_gx3_conrad_4x1
+./cice.build
+./cice.submit
+# After job finishes, check output
+cat test_output
+
+
+Additional Details:
+- The available options for '-t' are 'smoke', '10day', 'annual', and 'restart'.
+- For a 'restart' test, no baseline dataset needs to be generated or compared against.
+- The '-b' flag can be combined with '-bd' to specify the location where you 
+  want the baseline dataset to be written.  Without specifying '-bd', the
+  baseline dataset will be written to the default baseline directory found
+  in the env.<machine> file (CICE_MACHINE_BASELINE).
+- If neither '-b' or '-bd' are passed, the scripts will look for baseline datasets
+  in the default baseline directory found in the env.<machine> file (CICE_MACHINE_BASELINE).
+  If the '-bd' option is passed, the scripts will look for baseline datasets in the
+  location passed to the -bd argument.
+- To generate a baseline dataset for a specific version (for regression testing),
+  use '-b' in conjunction to '-bg <version_name>'.  The scripts will then place
+  the baseline dataset in $CICE_MACHINE_BASELINE/<version_name>/

README_v6

@@ -22,14 +22,17 @@ cd ~/mycase1
 A bit more information:
 
 create.case generates a case, use "create.case -h" for help with the tool.
-  -c is the case name and location (required)
+  -c is the case name and location
   -m is the machine name (required)
   -g is the resolution (default is gx3)
   -p is the task x thread/task values (default is 4x1)
   -s are comma separated optional env or namelist settings (default is "null")
   -t is the test name and location (cannot be used with -c).
   -b is used to specify that a baseline dataset is to be generated (only used with -t)
   -bd is used to specify the location of the baseline datasets (only used with -t)
+  -bg is used to specify the cice version name for generating baseline datasets (only used with -t)
+  -bc is used to specify the cice versoin name for comparison. I.e., the version name for the baseline dataset (only used with -t)
+  -testid is used to specify a test ID (used only with -t)
 
 Several files are placed in the case directory
  - env.${machine} defines the environment

configuration/scripts/tests/cice.10day.csh → configuration/scripts/cice.batch.csh

@@ -1,12 +1,15 @@
 #! /bin/csh -f
 
-echo ${0}
+if ( $1 != "" ) then
+  echo ${0} ${1}
+else
+  echo ${0}
+endif
 
 source ./cice.settings
 source ${CICE_CASEDIR}/env.${CICE_MACHINE} || exit 2
 
-set jobfile = cice.10day
-set subfile = cice.submit
+set jobfile = $1
 
 set ntasks = ${CICE_NTASKS}
 set nthrds = ${CICE_NTHRDS}
@@ -24,9 +27,6 @@ if ($ptile > ${maxtpn} / 2) @ ptile = ${maxtpn} / 2
 
 #==========================================
 
-# Create test script that runs cice.build, cice.run, and validates
-#==========================================
-
 cat >! ${jobfile} << EOF0
 #!/bin/csh -f 
 EOF0
@@ -48,6 +48,20 @@ cat >> ${jobfile} << EOFB
 #BSUB -P ${acct}
 EOFB
 
+#==========================================
+
+else if (${CICE_MACHINE} =~ cheyenne*) then
+cat >> ${jobfile} << EOFB
+#PBS -j oe 
+#PBS -m ae 
+#PBS -V
+#PBS -q regular
+#PBS -N ${CICE_CASENAME}
+#PBS -A ${CICE_ACCT}
+#PBS -l select=${nnodes}:ncpus=${ntasks}:mpiprocs=${ntasks}
+#PBS -l walltime=02:00:00
+EOFB
+
 else if (${CICE_MACHINE} =~ thunder* || ${CICE_MACHINE} =~ gordon* || ${CICE_MACHINE} =~ conrad*) then
 cat >> ${jobfile} << EOFB
 #PBS -N ${CICE_CASENAME}
@@ -78,88 +92,30 @@ EOFB
 else if (${CICE_MACHINE} =~ wolf*) then
 cat >> ${jobfile} << EOFB
 #SBATCH -J ${CICE_CASENAME}
-#SBATCH -t 0:10:00
+#SBATCH -t 0:45:00
 #SBATCH -A ${acct}
 #SBATCH -N ${nnodes}
 #SBATCH -e slurm%j.err
 #SBATCH -o slurm%j.out
-#SBATCH --mail-type FAIL
-#SBATCH [email protected]
+###SBATCH --mail-type END,FAIL
+###SBATCH [email protected]
 #SBATCH --qos=low
 EOFB
 
-endif
-
-cat >> ${jobfile} << EOF2
-cd ${CICE_CASEDIR}
-source ./cice.settings || exit 2
-source ./env.\${CICE_MACHINE} || exit 2
-
-# Compile the CICE code
-./cice.build
-set rc_build = \$?
-
-# Run the CICE model
-./cice.run
-set rc_run = \$?
-
-EOF2
-
-if ($1 != "") then
-cat >> ${jobfile} << EOF3
-  # Get the final output filename
-  foreach file (\${CICE_RUNDIR}/restart/*)
-    set test_data = \$file
-  end
-
-  set baseline_data = $1/\$test_data:t
-
-  if ( { cmp -s \$test_data \$baseline_data } ) then
-    set rc_valid = 0
-  else
-    set rc_valid = 1
-  endif
-EOF3
-endif
-
-cat >> ${jobfile} << EOF4
-
-if (\$rc_build == 0) then
-  echo "Build:      PASS"
-else
-  echo "Build:      FAIL"
-endif
-
-if (\$rc_run == 0) then
-  echo "Run:        PASS"
-else
-  echo "Run:        FAIL"
-endif
-EOF4
-
-if ($1 != "") then
-cat >> ${jobfile} << EOF5
+else if (${CICE_MACHINE} =~ pinto*) then
+cat >> ${jobfile} << EOFB
+#SBATCH -J ${CICE_CASENAME}
+#SBATCH -t 0:45:00
+#SBATCH -A ${acct}
+#SBATCH -N ${nnodes}
+#SBATCH -e slurm%j.err
+#SBATCH -o slurm%j.out
+###SBATCH --mail-type END,FAIL
+###SBATCH [email protected]
+#SBATCH --qos=standby
+EOFB
 
-if (\$rc_valid == 0) then
-  echo "Validation: PASS"
 else
-  echo "Validation: FAIL"
-endif
-EOF5
+  echo "${0} ERROR ${CICE_MACHINE} unknown"
+  exit -1
 endif
-
-#==========================================
-
-chmod +x ${jobfile}
-
-#==========================================
-
-cat >! ${subfile} << EOFS
-#!/bin/csh -f 
-
-${CICE_MACHINE_SUBMIT} ${jobfile}
-echo "\`date\` \${0}: ${CICE_CASENAME} job submitted"  >> ${CICE_CASEDIR}/README.test
-
-EOFS
-
-chmod +x ${subfile}

configuration/scripts/cice.build

@@ -80,11 +80,18 @@ gmake -j ${CICE_MACHINE_BLDTHRDS} VPFILE=Filepath EXEC=${CICE_RUNDIR}/cice CICE_
 if ($status != 0) then
   echo "${0}: COMPILE FAILED, see"
   echo "  cat ${CICE_OBJDIR}/${CICE_BLDLOG_FILE}"
+  if ( $?CICE_TEST ) then
+    # This is a test case.  Write output to test_output file
+    echo "FAIL $CICE_CASENAME build" >> ${CICE_CASEDIR}/test_output
+  endif
   exit 99
 endif
 
 if !(-d ${CICE_LOGDIR}) mkdir -p ${CICE_LOGDIR}
 cp -p ${CICE_BLDLOG_FILE} ${CICE_LOGDIR}/
 echo "`date` ${0}:${CICE_CASENAME} build completed ${CICE_BLDLOG_FILE}"  >> ${CICE_CASEDIR}/README.case
 echo "${0}: COMPILE SUCCESSFUL, ${CICE_LOGDIR}/${CICE_BLDLOG_FILE}"
+if ( $?CICE_TEST ) then
+  echo "PASS $CICE_CASENAME build" >> ${CICE_CASEDIR}/test_output
+endif
 

configuration/scripts/cice.launch.csh

@@ -0,0 +1,56 @@
+#! /bin/csh -f
+
+echo ${0}
+
+source ./cice.settings
+
+set jobfile = $1
+
+set ntasks = ${CICE_NTASKS}
+set nthrds = ${CICE_NTHRDS}
+
+#==========================================
+
+if (${CICE_MACHINE} =~ yellowstone*) then
+cat >> ${jobfile} << EOFR
+setenv MP_TASK_AFFINITY core:\${OMP_NUM_THREADS}
+mpirun.lsf ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ cheyenne*) then
+cat >> ${jobfile} << EOFR
+mpiexec_mpt -n ${ntasks} ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ thunder*) then
+cat >> ${jobfile} << EOFR
+mpiexec_mpt -np ${ntasks} omplace ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ gordon* || ${CICE_MACHINE} =~ conrad*) then
+cat >> ${jobfile} << EOFR
+aprun -n ${ntasks} -N ${ntasks} -d ${nthrds} ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ cori*) then
+cat >> ${jobfile} << EOFR
+srun -n ${ntasks} -c ${nthrds} ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ wolf*) then
+cat >> ${jobfile} << EOFR
+mpirun -np ${ntasks} ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+
+else if (${CICE_MACHINE} =~ pinto*) then
+cat >> ${jobfile} << EOFR
+mpirun -np ${ntasks} ./cice >&! \$CICE_RUNLOG_FILE
+EOFR
+#cat >> ${jobfile} << EOFR
+#srun -n ${ntasks} -c ${nthrds} ./cice >&! \$CICE_RUNLOG_FILE
+#EOFR
+
+else
+  echo "${0} ERROR ${CICE_MACHINE} unknown"
+  exit -1
+endif