Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add code changes for CI tests #1

Merged
merged 5 commits into from
Jan 22, 2021
Merged

add code changes for CI tests #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
580fabe
update ccpp control test forecast length to 24h
junwang-noaa Jan 22, 2021
59a5777
Merge remote-tracking branch 'upstream/develop' into fv3controltest
junwang-noaa Jan 22, 2021
c9ac422
remove rename command
junwang-noaa Jan 22, 2021
8f2bdfe
Add CI related changes
MinsukJi-NOAA Jan 22, 2021
462b6a8
Merge pull request #8 from MinsukJi-NOAA/fv3controltest
junwang-noaa Jan 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/ci/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
From noaaemc/ubuntu-hpc:v1
From noaaemc/ubuntu-hpc:v1.1

ENV HOME=/home/builder
COPY --chown=builder:builder . $HOME/ufs-weather-model
Expand Down
4 changes: 2 additions & 2 deletions tests/ci/ci.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ if [ $BUILD = "true" ]; then

elif [ $RUN == "true" ]; then

sudo docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:develop-20200713
sudo docker run -d -e test_case=${TEST_CASE} -v DataVolume:/home/builder/data/NEMSfv3gfs/develop-20201118 --name my-container ${IMG_NAME}
sudo docker run -d --rm -v DataVolume:/tmp minsukjinoaa/fv3-input-data:input-data-20210115
sudo docker run -d -e test_case=${TEST_CASE} -v DataVolume:/home/builder/data/NEMSfv3gfs/input-data-20210115 --name my-container ${IMG_NAME}

echo 'cache,rss,shmem' >memory_stat
sleep 3
Expand Down
40 changes: 27 additions & 13 deletions tests/fv3_conf/ccpp_control_run.IN
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,26 +11,40 @@ fi
echo "inputdir=$inputdir,NPX=$NPX"

UNIT_TEST=${UNIT_TEST:-false}
SUFFIX=${RT_SUFFIX}
if [ $WARM_START = .F. ]; then
cp -r @[INPUTDATA_ROOT]/${inputdir}/INPUT .
mkdir RESTART
else
mkdir INPUT RESTART

rsync -arv ../fv3_ccpp_control${RT_SUFFIX}/RESTART/ INPUT/
rm -f INPUT/fv_core.res.*
rm -f INPUT/fv_srf_wnd.res.*
rm -f INPUT/fv_tracer.res.*
rm -f INPUT/phy_data.*
rm -f INPUT/sfc_data.*
cd INPUT
rename 20161004.000000. '' 20161004.000000.*
cp ../../fv3_ccpp_control${RT_SUFFIX}/INPUT/grid_spec.nc .
cp ../../fv3_ccpp_control${RT_SUFFIX}/INPUT/*_grid.tile*.nc .
cp ../../fv3_ccpp_control${RT_SUFFIX}/INPUT/oro_data.tile*.nc .
if [[ $IAU_INC_FILES != '' ]] || [[ $READ_INCREMENT = '.T.' ]]; then
cp ../../fv3_ccpp_control${RT_SUFFIX}/INPUT/fv3_increment.nc .
if [[ ${UNIT_TEST} == true ]]; then
SUFFIX=${BL_SUFFIX}
fi

rsync -arv ../fv3_ccpp_control${SUFFIX}/RESTART/ INPUT/
if [[ $IAU_INC_FILES = 'fv3_increment.nc' ]] || [[ $READ_INCREMENT = '.T.' ]]; then
#read_inc and iau tests restart from fh=24
cp ../fv3_ccpp_control${SUFFIX}/RESTART/coupler.res INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/RESTART/fv_core.res.* INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/RESTART/fv_srf_wnd.res.* INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/RESTART/fv_tracer.* INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/RESTART/phy_data.* INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/RESTART/sfc_data.* INPUT/.
cp ../fv3_ccpp_control${SUFFIX}/INPUT/fv3_increment.nc INPUT/.
cd INPUT
else
#restart test start from fh=12
cp ../fv3_ccpp_control${SUFFIX}/RESTART/20161003.120000.* INPUT/.
cd INPUT
for RFILE in 20161003.120000.*; do
[ -e $RFILE ] || exit 1
mv $RFILE ${RFILE#20161003.120000.}
done
fi
cp ../../fv3_ccpp_control${SUFFIX}/INPUT/grid_spec.nc .
cp ../../fv3_ccpp_control${SUFFIX}/INPUT/*_grid.tile*.nc .
cp ../../fv3_ccpp_control${SUFFIX}/INPUT/oro_data.tile*.nc .
cd ..

fi
Expand Down
1 change: 0 additions & 1 deletion tests/tests/fv3_ccpp_2threads
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \


export_fv3
export FHMAX=48
export THRD=2
export TASKS=$TASKS_thrd
export TPN=$TPN_thrd
Expand Down
15 changes: 1 addition & 14 deletions tests/tests/fv3_ccpp_control
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,6 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
phyf024.tile4.nc \
phyf024.tile5.nc \
phyf024.tile6.nc \
phyf048.tile1.nc \
phyf048.tile2.nc \
phyf048.tile3.nc \
phyf048.tile4.nc \
phyf048.tile5.nc \
phyf048.tile6.nc \
dynf000.tile1.nc \
dynf000.tile2.nc \
dynf000.tile3.nc \
Expand All @@ -44,12 +38,6 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \
dynf024.tile4.nc \
dynf024.tile5.nc \
dynf024.tile6.nc \
dynf048.tile1.nc \
dynf048.tile2.nc \
dynf048.tile3.nc \
dynf048.tile4.nc \
dynf048.tile5.nc \
dynf048.tile6.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
Expand Down Expand Up @@ -85,8 +73,7 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \


export_fv3
export FHMAX=48
export RESTART_INTERVAL=24
export RESTART_INTERVAL=12

export FV3_RUN=ccpp_control_run.IN
export CCPP_SUITE=FV3_GFS_2017
Expand Down
1 change: 0 additions & 1 deletion tests/tests/fv3_ccpp_decomp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,6 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \


export_fv3
export FHMAX=48

export INPES=6
export JNPES=4
Expand Down
33 changes: 14 additions & 19 deletions tests/tests/fv3_ccpp_restart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,18 @@ export TEST_DESCR="Compare FV3 CCPP restart results with previous trunk version"

export CNTL_DIR=fv3_control

export LIST_FILES="atmos_4xdaily.tile1.nc \
atmos_4xdaily.tile2.nc \
atmos_4xdaily.tile3.nc \
atmos_4xdaily.tile4.nc \
atmos_4xdaily.tile5.nc \
atmos_4xdaily.tile6.nc \
phyf048.tile1.nc \
phyf048.tile2.nc \
phyf048.tile3.nc \
phyf048.tile4.nc \
phyf048.tile5.nc \
phyf048.tile6.nc \
dynf048.tile1.nc \
dynf048.tile2.nc \
dynf048.tile3.nc \
dynf048.tile4.nc \
dynf048.tile5.nc \
dynf048.tile6.nc \
export LIST_FILES="phyf024.tile1.nc \
phyf024.tile2.nc \
phyf024.tile3.nc \
phyf024.tile4.nc \
phyf024.tile5.nc \
phyf024.tile6.nc \
dynf024.tile1.nc \
dynf024.tile2.nc \
dynf024.tile3.nc \
dynf024.tile4.nc \
dynf024.tile5.nc \
dynf024.tile6.nc \
RESTART/coupler.res \
RESTART/fv_core.res.nc \
RESTART/fv_core.res.tile1.nc \
Expand Down Expand Up @@ -61,14 +55,15 @@ export LIST_FILES="atmos_4xdaily.tile1.nc \


export_fv3
export FHROT=12
export FHMAX=24

export WARM_START=.T.
export NGGPS_IC=.F.
export EXTERNAL_IC=.F.
export MAKE_NH=.F.
export MOUNTAIN=.T.
export NA_INIT=0
export FHMAX=48
export FDIAG=3
export NSTF_NAME=2,0,1,0,5

Expand Down
31 changes: 2 additions & 29 deletions tests/utest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,8 +136,6 @@ run_utests() {
case $rc in
std_base)
CREATE_BASELINE=true
RTPWD=$baseline_location
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env
export UNIT_TEST=true
Expand All @@ -147,8 +145,6 @@ run_utests() {
;;
std)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
cat <<-EOF > ${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env
export UNIT_TEST=true
Expand All @@ -157,8 +153,6 @@ run_utests() {
;;
thr)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
THRD=2
# INPES is sometimes odd, so use JNPES. Make sure JNPES is divisible by THRD
Expand All @@ -178,8 +172,6 @@ run_utests() {
;;
mpi)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
JNPES=$(( JNPES/2 ))
WRITE_GROUP=2
Expand All @@ -198,8 +190,6 @@ run_utests() {
;;
dcp)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
temp=$INPES
INPES=$JNPES
Expand All @@ -213,8 +203,6 @@ run_utests() {
;;
rst) # this is not going to work for regional model
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_std_base
# Set up date and time of restart files for restart run
RESTART_FILE_PREFIX="${SYEAR}${SMONTH}${SDAY}.$(printf "%02d" $(( SHOUR + FHMAX/2 )))0000"
Expand Down Expand Up @@ -277,8 +265,6 @@ run_utests() {
;;
bit_base)
CREATE_BASELINE=true
RTPWD=$baseline_location
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_bit_base
comp_nm=bit
cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env
Expand All @@ -288,8 +274,6 @@ run_utests() {
;;
bit)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_bit_base
comp_nm=bit
cat <<-EOF >${RUNDIR_ROOT}/unit_test${RT_SUFFIX}.env
Expand All @@ -299,8 +283,6 @@ run_utests() {
;;
dbg_base)
CREATE_BASELINE=true
RTPWD=$baseline_location
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_dbg_base
comp_nm=dbg
WLCLK=30
Expand All @@ -312,8 +294,6 @@ run_utests() {
;;
dbg)
CREATE_BASELINE=false
RTPWD=$NEW_BASELINE
INPUTDATA_ROOT=$RTPWD
BL_SUFFIX=_dbg_base
comp_nm=dbg
WLCLK=30
Expand All @@ -324,7 +304,6 @@ run_utests() {
EOF
;;
esac
INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20201220/

cat <<- EOF > ${RUNDIR_ROOT}/run_test${RT_SUFFIX}.env
export MACHINE_ID=${MACHINE_ID}
Expand Down Expand Up @@ -723,18 +702,12 @@ fi

mkdir -p ${STMP}/${USER}
NEW_BASELINE=${STMP}/${USER}/FV3_UT/UNIT_TEST
if [[ $MACHINE_ID == hera.* || $MACHINE_ID == orion.* ]]; then
baseline_location=$DISKNM/NEMSfv3gfs/develop-20201118/${RT_COMPILER^^}
else
baseline_location=$DISKNM/NEMSfv3gfs/develop-20201118
fi
RTPWD=$baseline_location
INPUTDATA_ROOT=$RTPWD
RTPWD=${NEW_BASELINE}
INPUTDATA_ROOT=${INPUTDATA_ROOT:-$DISKNM/NEMSfv3gfs/input-data-20210115}
INPUTDATA_ROOT_WW3=${INPUTDATA_ROOT}/WW3_input_data_20201220/

rm -rf $NEW_BASELINE
mkdir -p $NEW_BASELINE
ln -s ${RTPWD}/FV3_input_data ${NEW_BASELINE}

rm -f fail_unit_test ${unittest_log}

Expand Down