diff --git a/README_fates_emerald_api_expert_only b/README_fates_emerald_api_expert_only index d1ef7bea9b..0a4e4badf1 100644 --- a/README_fates_emerald_api_expert_only +++ b/README_fates_emerald_api_expert_only @@ -1 +1,74 @@ -To be added soon. +$CTSMROOT/README_fates_emerald_api_expert_only 03/06/2020 + +################################################################################# +# This document explains how to update the model codes and +# how to create atmospheric forcing and inputdata for new sites +# (No technical support will be promised, use at your own risk!) +# See also Yeliz's notes: https://ctsm-docs.readthedocs.io/en/latest/ for additional details +################################################################################# + +######## Two important things to watch out first!!! +######## Two important things to watch out first!!! +######## Two important things to watch out first!!! +(1) Module environment on your supercomputer: DON'T load any module beforehand, except "StdEvn"! +(2) Git version control: + CLM-FATES have very complex and sperate repository system: + -> ctsm: always create your own branch, do not touch master! + -> cime: remember to create your own branch in cime folder also, to record all the changes you make. + -> fates: remember to create your own branch in fates folder also, to record all the changes you make. + If you do not create your own branch for "cime" and "fates", when you run "./manage_externals/checkout_externals", your previous "cime" and "fates" will be overwritten. + + +######## Download ctsm and update ctsm-fates (Only important steps are illustrated here!!!) +**** If you are already using fates_emerald_api, skip this! **** +e.g.: + git remote add ctsm_ncar https://github.com/ESCOMP/CTSM.git # set a remote name for NCAR CTSM git repository + git fetch ctsm_ncar # get the most recent updates of CTSM + git merge ctsm_ncar/master # update your current branch with the newest CTSM version + git merge ctsm_ncar/fates_next_api # update your current branch with the newest CLM-FATES interface + +##To update fates, you need to go to fates folder +e.g.: + git remote add NGEET_fates https://github.com/NGEET/fates.git + git fetch NGEET_fates # get the most recent updates of FATES + git merge NGEET_fates.git/master # update your current branch with the newest FATES + +######## Adapt ctsm/cime to specific machine (e.g., SAGA and FRAM) +**** If you are already using fates_emerald_api, skip this! **** +Following file need to be modified +..../cime/config/cesm/machines/config_batch.xml +..../cime/config/cesm/machines/config_compilers.xml +..../cime/config/cesm/machines/config_machines.xml + +See example here, you need look for the setup +https://github.com/NordicESMhub/cime/blob/cime5.6.10_noresm2_1_rel_06/config/cesm/machines/config_compilers.xml +https://github.com/NordicESMhub/cime/blob/cime5.6.10_noresm2_1_rel_06/config/cesm/machines/config_machines.xml +https://github.com/NordicESMhub/cime/blob/cime5.6.10_noresm2_1_rel_06/config/cesm/machines/config_batch.xml + + +**** If you want to set up new sites, please follow the following steps **** +######## Make script grids (see "prepare_inputdata.sh" in ./tools) +######## Make domain file (see "prepare_inputdata.sh" in ./tools) +######## Make mapping file (see "prepare_inputdata.sh" in ./tools) +######## Make surface data file (see "prepare_inputdata.sh" in ./tools) +######## Make and modify FATES parameter file: + ## Use existing tools: + ./src/fates/tools/FatesPFTIndexSwapper.py # Select PFTs interested. + ./src/fates/tools/modify_fates_paramfile.py # Modify PFT parameters + ## To use these tools, netcdf and python modules need to be loaded. + module load netCDF-Fortran/4.4.4-intel-2018b + module load Python/3.6.6-intel-2018b + ./FatesPFTIndexSwapper.py --pft-indices=2,6,9,10 --fin=****.nc --fout=****.nc + ## The default FATES parameter file is in ./src/fates/parameter_files + ## To create a netcdf parameter file: ncgen -o ****.nc fates_params_default.cdl +######## Make atmospheric forcing for new sites (see "prepare_atm_forcing_****.ncl" in ./tools) + +######## Set each sites as a unique grid type. The following file need to be modified: +~/ctsm/cime/config/cesm/config_grids.xml +~/ctsm/cime/src/components/data_comps/datm/cime_config/namelist_definition_datm.xml +~/ctsm/bld/namelist_files/namelist_defaults_clm4_5.xml + +######## Set new atmospheric forcing for the sites +~/ctsm/cime/src/components/data_comps/datm/cime_config/config_component.xml (for different climate forcing) +~/ctsm/cime_config/config_component.xml (for different climate forcing) + diff --git a/bld/namelist_files/namelist_defaults_clm4_5.xml b/bld/namelist_files/namelist_defaults_clm4_5.xml index dc15c3b97d..ed4307b2f3 100644 --- a/bld/namelist_files/namelist_defaults_clm4_5.xml +++ b/bld/namelist_files/namelist_defaults_clm4_5.xml @@ -796,6 +796,24 @@ lnd/clm2/surfdata_map/fates_platform/BOR2/surfdata_BOR2_simyr2000.nc lnd/clm2/surfdata_map/fates_platform/BOR3/surfdata_BOR3_simyr2000.nc lnd/clm2/surfdata_map/fates_platform/BOR4/surfdata_BOR4_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/LYG/surfdata_LYG_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/BUO/surfdata_BUO_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/HAV/surfdata_HAV_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/SKO/surfdata_SKO_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/VIKE/surfdata_VIKE_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/JOAS/surfdata_JOAS_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/LIAH/surfdata_LIAH_simyr2000.nc + +lnd/clm2/surfdata_map/fates_platform/FINN/surfdata_FINN_simyr2000.nc + + @@ -1155,6 +1173,14 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_0.9x1.25_SSP5-3.4_78pfts_CMIP6_ nn nn nn +nn +nn +nn +nn +nn +nn +nn +nn @@ -1204,7 +1230,14 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_0.9x1.25_SSP5-3.4_78pfts_CMIP6_ nn nn nn - +nn +nn +nn +nn +nn +nn +nn +nn @@ -1303,6 +1336,14 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_0.9x1.25_SSP5-3.4_78pfts_CMIP6_ nn nn nn +nn +nn +nn +nn +nn +nn +nn +nn 2015 diff --git a/tools/mksurfdata_map/src/mkdomainMod.F90 b/tools/mksurfdata_map/src/mkdomainMod.F90 index 1a66c0f4c5..3aaed88531 100644 --- a/tools/mksurfdata_map/src/mkdomainMod.F90 +++ b/tools/mksurfdata_map/src/mkdomainMod.F90 @@ -257,6 +257,7 @@ logical function domain_read_map(domain, fname) ! Read domain file and compute stuff as needed + call check_ret(nf_open(fname, 0, ncid), subname) ! Assume unstructured grid diff --git a/tools/prepare_atm_forcing_gswp3.ncl b/tools/prepare_atm_forcing_gswp3.ncl new file mode 100644 index 0000000000..287aee91e3 --- /dev/null +++ b/tools/prepare_atm_forcing_gswp3.ncl @@ -0,0 +1,159 @@ +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/contributed.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/csm/popRemap.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/esmf/ESMF_regridding.ncl" +load "$NCARG_ROOT/lib/ncarg/nclscripts/contrib/cd_string.ncl" + +begin + +;######### SeedClim Sites +; plot_lat=(/61.0243,60.8231,60.8328,60.9335,60.8203,60.8760,61.0866,60.5445,61.0355,60.8803,60.6652,60.6901/) +; plot_lon=(/8.12343,7.27596,7.17561,6.41504,8.70466,7.17666,6.63028,6.51468,9.07876,7.16982,6.33738,5.96487/) +; plot_name=(/"ALP1","ALP2","ALP3","ALP4","SUB1","SUB2","SUB3","SUB4","BOR1","BOR2","BOR3","BOR4"/) + +;######### Landpress Sites +; plot_lat=(/60.70084,65.83677,64.779,65.79602/) +; plot_lon=(/5.092566,12.224506,11.2193,12.219299/) +; plot_name=(/"LYG","BUO","HAV","SKO"/) + +;######### Three-D Sites + plot_lat=(/60.88019,60.86183,60.85994/) + plot_lon=(/7.16990,7.16800,7.19504/) + plot_name=(/"VIKE","JOAS","LIAH"/) + +;######### Finnmark Site +; plot_lat=(/69.341088/) +; plot_lon=(/25.293524/) +; plot_name=(/"FINN"/) + + mon = ispan(0,11,1) + mon@units = "month" + month =cd_string(mon,"%N") + print(month) + + do p=0,2,1 + clim_output = "/cluster/shared/noresm/inputdata/atm/datm7/fates_platform/"+plot_name(p)+"/" + clim_input = "/cluster/shared/noresm/inputdata/atm/datm7/atm_forcing.datm7.GSWP3.0.5d.v1.c170516/" + system("mkdir -p "+ clim_output) + xfloor=plot_lon(p)-0.005 + xceil =plot_lon(p)+0.005 + yfloor=plot_lat(p)-0.005 + yceil =plot_lat(p)+0.005 + print(yceil) + print(xceil) + + do year=1901,2010,1 + do m=0,11,1 + + frsds=addfile(clim_input+"/Solar/clmforc.GSWP3.c2011.0.5x0.5.Solr."+year+"-"+month(m)+".nc","r") + fprec=addfile(clim_input+"/Precip/clmforc.GSWP3.c2011.0.5x0.5.Prec."+year+"-"+month(m)+".nc","r") + ftphw=addfile(clim_input+"/TPHWL/clmforc.GSWP3.c2011.0.5x0.5.TPQWL."+year+"-"+month(m)+".nc","r") + + rsds=frsds->FSDS(:,:,:) + prec=fprec->PRECTmms(:,:,:) + psrf=ftphw->PSRF(:,:,:) + tbot=ftphw->TBOT(:,:,:) + wind=ftphw->WIND(:,:,:) + qbot=ftphw->QBOT(:,:,:) + flds=ftphw->FLDS(:,:,:) + + lon=frsds->LONGXY(0,:) + lat=frsds->LATIXY(:,0) + time=frsds->time(:) + edgew =frsds->EDGEW + edgee =frsds->EDGEE + edges =frsds->EDGES + edgen =frsds->EDGEN + longxy =frsds->LONGXY + latixy =frsds->LATIXY + + loni=ind_nearest_coord(plot_lon(p), lon, 0) + latj=ind_nearest_coord(plot_lat(p), lat, 0) + print(loni) + print(latj) + +;########## Creat forcing in netcdf format + + system("rm -f "+clim_output+"clm1pt_"+plot_name(p)+"_"+year+"-"+month(m)+".nc") ; remove if exists + fclim = addfile(clim_output+"clm1pt_"+plot_name(p)+"_"+year+"-"+month(m)+".nc","c") + ntim = dimsizes(time) ; get dimension sizes + nlat = 1 + nlon = 1 + nscalar = 1 + + setfileoption(fclim,"DefineMode",True) + + fAtt = True ; assign file attributes + fAtt@case_title = "GSWP3v1: norway " + fAtt@conventions = "CF-1.0" + fAtt@title = "CLM single point datm input data" + fAtt@history = "Original data from GSWP3v1 data" + fAtt@creation_date = systemfunc ("date") + fileattdef( fclim, fAtt ) ; copy file attributes + + dimNames = (/"scalar","lon","lat","time"/) + dimSizes = (/ nscalar, nlon, nlat, -1 /) + dimUnlim = (/ False, False, False, True/) + filedimdef(fclim,dimNames,dimSizes,dimUnlim) + + filevardef(fclim, "EDGEW" ,typeof(edgew),getvardims(edgew)) + filevardef(fclim, "EDGEE" ,typeof(edgee),getvardims(edgee)) + filevardef(fclim, "EDGES" ,typeof(edges),getvardims(edges)) + filevardef(fclim, "EDGEN" ,typeof(edgen),getvardims(edgen)) + filevardef(fclim, "LONGXY" ,typeof(longxy) ,getvardims(longxy)) + filevardef(fclim, "LATIXY" ,typeof(latixy) ,getvardims(latixy)) + filevardef(fclim, "time",typeof(time),getvardims(time)) + filevardef(fclim, "FSDS",typeof(rsds),getvardims(rsds)) + filevardef(fclim, "PRECTmms",typeof(prec),getvardims(prec)) + filevardef(fclim, "TBOT",typeof(tbot),getvardims(tbot)) + filevardef(fclim, "WIND",typeof(wind),getvardims(wind)) + filevardef(fclim, "PSRF",typeof(psrf),getvardims(psrf)) + filevardef(fclim, "QBOT",typeof(qbot),getvardims(qbot)) + filevardef(fclim, "FLDS",typeof(flds),getvardims(flds)) + + filevarattdef(fclim,"EDGEW",edgew) + filevarattdef(fclim,"EDGEE",edgee) + filevarattdef(fclim,"EDGES",edges) + filevarattdef(fclim,"EDGEN",edgen) + filevarattdef(fclim,"LONGXY",longxy) + filevarattdef(fclim,"LATIXY",latixy) + filevarattdef(fclim,"time",time) + filevarattdef(fclim,"FSDS",rsds) + filevarattdef(fclim,"PRECTmms",prec) + filevarattdef(fclim,"TBOT",tbot) + filevarattdef(fclim,"WIND",wind) + filevarattdef(fclim,"PSRF",psrf) + filevarattdef(fclim,"QBOT",qbot) + filevarattdef(fclim,"FLDS",flds) + + setfileoption(fclim,"DefineMode",False) + + print(time) + fclim->time = (/time/) ; "(/", "/)" syntax tells NCL to only output the data values to the predefined locations on the file. + fclim->EDGEW = (/xfloor/) + fclim->EDGEE = (/xceil/) + fclim->EDGES = (/yfloor/) + fclim->EDGEN = (/yceil/) + fclim->LONGXY = (/plot_lon(p)/) + fclim->LATIXY = (/plot_lat(p)/) + fclim->FSDS = (/rsds(:,latj,loni)/) + fclim->PRECTmms = (/prec(:,latj,loni)/) ; !!!! time variable can be modified in the attributes of the variable is also copied. make sure to use (/ /) to only copy data. + fclim->TBOT = (/tbot(:,latj,loni)/) + fclim->WIND = (/wind(:,latj,loni)/) + fclim->PSRF = (/psrf(:,latj,loni)/) + fclim->QBOT = (/qbot(:,latj,loni)/) + fclim->FLDS = (/flds(:,latj,loni)/) + + delete(time) + delete(rsds) + delete(prec) + delete(tbot) + delete(qbot) + delete(wind) + delete(psrf) + delete(flds) + + end do + end do + end do +end + diff --git a/tools/prepare_inputdata.sh b/tools/prepare_inputdata.sh new file mode 100755 index 0000000000..3ff8022358 --- /dev/null +++ b/tools/prepare_inputdata.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +#SBATCH --account=nn2806k +#SBATCH --job-name=mkmapdata +#SBATCH --mem-per-cpu=256G --partition=bigmem +#SBATCH --ntasks=1 +#SBATCH --time=07:00:00 + +date="200605" # Need to be the same as the running date + +######### SeedClim Sites +#plotlat=(61.0243 60.8231 60.8328 60.9335 60.8203 60.8760 61.0866 60.5445 61.0355 60.8803 60.6652 60.6901) +#plotlon=(8.12343 7.27596 7.17561 6.41504 8.70466 7.17666 6.63028 6.51468 9.07876 7.16982 6.33738 5.96487) +#plotname=(ALP1 ALP2 ALP3 ALP4 SUB1 SUB2 SUB3 SUB4 BOR1 BOR2 BOR3 BOR4) + +######### Landpress Sites +#plotlat=(60.70084 65.83677 64.779 65.79602) +#plotlon=(5.092566 12.224506 11.2193 12.219299) +#plotname=(LYG BUO HAV SKO) + +######### Three-D Sites +#plotlat=(60.88019 60.86183 60.85994) +#plotlon=(7.16990 7.16800 7.19504) +#plotname=(VIKE JOAS LIAH) + +######### Finnmark Site +plotlat=(69.341088) +plotlon=(25.293524) +plotname=(FINN) + +creat_script="T" # T or F, switch for creating script grid or not +creat_domain="T" # T or F, switch for creating domain file +creat_mapping="T" # T or F, switch for creating mapping file +creat_surfdat="T" # T or F, switch for creating surface data file +run_case="F" # T or F, switch for running many clm experiments automatically. "T" can only be used when all the inputdata are ready!!!! +run_case_first="F" # T or F, swtich for creating, building and submitting short test runs +run_case_second="F" # T or F, swtich for running long experiments + +#0 1 2 3 4 5 6 7 8 9 10 11 +for i in 0 +do + +######## Make script grids: +if [ ${creat_script} == "T" ] +then + + module purge + module load NCL/6.6.2-intel-2019b + cd ~/ctsm/tools/mkmapdata + mkdir -p /cluster/shared/noresm/inputdata/share/scripgrids/fates_platform/${plotname[i]} + + echo $i + echo ${plotlat[$i]} ${plotlon[$i]} + ./mknoocnmap.pl -centerpoint ${plotlat[i]},${plotlon[i]} -name ${plotname[i]} -dx 0.01 -dy 0.01 + mv ~/ctsm/tools/mkmapgrids/*${plotname[i]}*.nc /cluster/shared/noresm/inputdata/share/scripgrids/fates_platform/${plotname[i]}/ + +fi + +######## Make domain file: +if [ ${creat_domain} == "T" ] +then + module purge + cd ~/ctsm/cime/tools/mapping/gen_domain_files + mkdir -p /cluster/shared/noresm/inputdata/share/domains/fates_platform/${plotname[i]} + + #### Compile (Only need to be run at the first time, better to run separately) + #cd src/ + #../../../configure --macros-format Makefile --mpilib mpi-serial --machine saga + #. ./.env_mach_specific.sh ; gmake + + #### Run + . ./src/.env_mach_specific.sh + ./gen_domain -m /cluster/shared/noresm/inputdata/share/scripgrids/fates_platform/${plotname[i]}/map_${plotname[i]}_noocean_to_${plotname[i]}_nomask_aave_da_${date}.nc -o ${plotname[i]} -l ${plotname[i]} + + mv domain.lnd.${plotname[i]}_${plotname[i]}.${date}.nc domain.lnd.${plotname[i]}.${date}.nc + mv domain*${plotname[i]}*.nc /cluster/shared/noresm/inputdata/share/domains/fates_platform/${plotname[i]}/ + +fi + +######## Make mapping file: +if [ ${creat_mapping} == "T" ] +then + + module purge + cd ~/ctsm/tools/mkmapdata + mkdir -p /cluster/shared/noresm/inputdata/lnd/clm2/mappingdata/maps/fates_platform/${plotname[i]}/ + #### Modify regridbatch.sh. This has been done in "fates_emerald_api". + #### Run regridbatch.sh + ./regridbatch.sh 1x1_${plotname[i]} /cluster/shared/noresm/inputdata/share/scripgrids/fates_platform/${plotname[i]}/SCRIPgrid_${plotname[i]}_nomask_c${date}.nc + mv map*${plotname[i]}*.nc /cluster/shared/noresm/inputdata/lnd/clm2/mappingdata/maps/fates_platform/${plotname[i]}/ + +fi + +######## Make surface data file: +if [ ${creat_surfdat} == "T" ] +then + + module purge + module load netCDF-Fortran/4.5.2-iimpi-2019b + cd ~/ctsm/tools/mksurfdata_map + mkdir -p /cluster/shared/noresm/inputdata/lnd/clm2/surfdata_map/fates_platform/${plotname[i]} + + #### Compile (Only need to be run at the first time, better to run separately) + #module load netCDF-Fortran/4.5.2-iimpi-2019b + #Modify Makefile.common. This has been done in "fates_emerald_api". + #gmake clean + #gmake + + #### Run + ./mksurfdata.pl -no-crop -res usrspec -usr_gname 1x1_${plotname[i]} -usr_gdate ${date} -usr_mapdir /cluster/shared/noresm/inputdata/lnd/clm2/mappingdata/maps/fates_platform/${plotname[i]} -dinlc /cluster/shared/noresm/inputdata -hirespft -years "2005" -allownofile + mv surfdata_1x1_${plotname[i]}_hist_16pfts_Irrig_CMIP6_simyr2005_c${date}.nc surfdata_${plotname[i]}_simyr2000.nc + mv surfdata*${plotname[i]}* /cluster/shared/noresm/inputdata/lnd/clm2/surfdata_map/fates_platform/${plotname[i]} + +fi + +######## Create a case +if [ ${run_case} == "T" ] +then + + module purge + if [ ${run_case_first} == "T" ] + then + cd ~/ctsm/cime/scripts + ./create_newcase --case ../../../ctsm_cases/2000FATES_seedclim_${plotname[i]} --compset 2000_DATM%1PTGSWP3_CLM50%FATES_SICE_SOCN_MOSART_SGLC_SWAV --res 1x1_${plotname[i]} --machine saga --run-unsupported --project nn2806k + +######## Build the case and run the test + cd ~/ctsm_cases/2000FATES_seedclim_${plotname[i]} + ./case.setup + ./case.build + ./case.submit + fi + +######## Run the case for longer time + if [ ${run_case_second} == "T" ] + then + cd ~/ctsm_cases/2000FATES_seedclim_${plotname[i]} + ./xmlchange --file env_run.xml --id STOP_OPTION --val nyears # set up the time unit (e.g., nyears, nmonths, ndays). + ./xmlchange --file env_run.xml --id STOP_N --val 200 # set up the length of the simulation. + #./xmlchange --file env_run.xml --id CONTINUE_RUN --val TRUE # if you want to continue your simulation from the restart file, set it to TRUE. + ./xmlchange --file env_run.xml --id RESUBMIT --val 9 # set up how many times you want to resubmit your simulation. + ./xmlchange --file env_workflow.xml --id JOB_WALLCLOCK_TIME --val 08:00:00 # set up longer queue time for runing the simulation. + ./xmlchange --file env_workflow.xml --id JOB_QUEUE --val bigmem # set up which queue to be used. Both "normal" and "bigmem" can be used depending on their availability. + ./case.submit + fi + +fi + +done +