Merge tag 'ctsm5.1.dev103' into reduce_redundancy

Fix accumulation variables when changing model time step Accumulation variables (e.g., 1-day or 10-day averages) were writing and reading their accumulation period (expressed in time steps) to the restart file. This caused incorrect behavior when changing the model time step relative to what was used to create the initial conditions file (typically a 30-minute time step). So, for example, if you are using a 15-minute time step with an initial conditions file that originated from a run with a 30-minute time step (at some point in its history), then an average that was supposed to be 10-day instead becomes 5-day; an average that was supposed to be 1-day becomes 12-hour, etc. (The issue is that the number of time steps in the averaging period was staying fixed rather than the actual amount of time staying fixed.) For our out-of-the-box initial conditions files, this only impacts runs that use something other than a 30-minute time step. Typically this situation arises in configurations with an active atmospheric model that is running at higher resolution than approximately 1 degree. It appears that the biggest impacts are on VOC emissions and in BGC runs; we expect the impact to be small (but still non-zero) in prescribed phenology (SP) runs that don't use VOC emissions. This tag fixes this issue by no longer writing or reading accumulation variables' PERIOD to / from the restart file: this isn't actually needed on the restart file. See some discussion in ESCOMP#1789 for more details, and see ESCOMP#1802 (comment) for some discussion of outstanding weirdness that can result for accumulation variables when changing the model time step. The summary of that comment is: There could be some weirdness at the start of a run, but at least for a startup or hybrid run, that weirdness should work itself out within about the first averaging period. A branch or restart run could have some longer-term potential weirdness, so for now I think we should recommend that people NOT change the time step on a branch or restart run. With (significant?) additional work, we could probably avoid this additional weirdness, but my feeling is that it isn't worth the effort right now. In any case, I feel like my proposed fix will bring things much closer to being correct than they currently are when changing the time step. Resolved conflict: python/ctsm/modify_fsurdat/fsurdat_modifier.py
slevis-lmwg · Jul 14, 2022 · e2f8e01 · e2f8e01
2 parents 3625f8f + bc7f683
commit e2f8e01
Show file tree

Hide file tree

Showing 185 changed files with 15,207 additions and 6,154 deletions.
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
@@ -0,0 +1,4 @@
+# Ran python directory through black python formatter
+4cd83cb3ee6d85eb909403487abf5eeaf4d98911
+0aa2957c1f8603c63fa30b11295c06cfddff44a5
+2cdb380febb274478e84cd90945aee93f29fa2e6
diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml
@@ -0,0 +1,21 @@
+name: black check on push and PR
+#
+# Run the python formatting in check mode
+#
+on: [push, pull_request]
+
+jobs:
+  black-check:
+     runs-on: ubuntu-latest
+     steps:
+        # Checkout the code
+        - uses: actions/checkout@v2
+        # Use the latest stable version of the github action
+        - uses: psf/black@stable
+          with:
+             # Use options and version identical to the conda environment
+             # Using pyproject.toml makes sure this testing is consistent with our python directory testing
+             options: "--check --config python/pyproject.toml"
+             src:     "./python"
+             # Version should be coordinated with the ctsm_py conda environment under the python directory
+             version: "22.3.0"
diff --git a/Externals.cfg b/Externals.cfg
@@ -34,7 +34,7 @@ hash = 34723c2
 required = True
 
 [ccs_config]
-tag = ccs_config_cesm0.0.15
+tag = ccs_config_cesm0.0.36
 protocol = git
 repo_url = https://github.com/ESMCI/ccs_config_cesm.git
 local_path = ccs_config
@@ -44,11 +44,11 @@ required = True
 local_path = cime
 protocol = git
 repo_url = https://github.com/ESMCI/cime
-tag = cime6.0.15
+tag = cime6.0.27
 required = True
 
 [cmeps]
-tag = cmeps0.13.47
+tag = cmeps0.13.68
 protocol = git
 repo_url = https://github.com/ESCOMP/CMEPS.git
 local_path = components/cmeps
@@ -63,14 +63,14 @@ externals =  Externals_CDEPS.cfg
 required = True
 
 [cpl7]
-tag = cpl7.0.12
+tag = cpl7.0.13
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_CPL7andDataComps
 local_path = components/cpl7
 required = True
 
 [share]
-tag = share1.0.10
+tag = share1.0.12
 protocol = git
 repo_url = https://github.com/ESCOMP/CESM_share
 local_path = share
@@ -84,7 +84,7 @@ local_path = libraries/mct
 required = True
 
 [parallelio]
-tag = pio2_5_6
+tag = pio2_5_7
 protocol = git
 repo_url = https://github.com/NCAR/ParallelIO
 local_path = libraries/parallelio

diff --git a/Externals_CLM.cfg b/Externals_CLM.cfg
@@ -2,7 +2,7 @@
 local_path = src/fates
 protocol = git
 repo_url = https://github.com/NGEET/fates
-tag = sci.1.55.4_api.22.1.0
+tag = sci.1.57.4_api.24.0.0
 required = True
 
 [externals_description]

diff --git a/README b/README
@@ -45,6 +45,7 @@ test -------------- CTSM Testing scripts for CTSM offline tools.
 tools ------------- CTSM Offline tools to prepare input datasets and process output.
 cime_config ------- Configuration files of cime for compsets and CTSM settings
 manage_externals -- Script to manage the external source directories
+manage_python_env - Script to setup the python environment for CTSM python tools using conda
 python ------------ Some python modules mostly for use by run_sys_tests (but could be used elsewhere l
 
 Directory structure only for a CTSM checkout:

diff --git a/bld/CLMBuildNamelist.pm b/bld/CLMBuildNamelist.pm
@@ -741,7 +741,7 @@ sub setup_cmdl_fates_mode {
 
     $var = "use_fates";
     if ( &value_is_true($nl_flags->{$var}) ) {
-      # This section is a place-holder to test for modules that are not allowed with ED
+      # This section is a place-holder to test for modules that are not allowed with FATES
       # the defaults which are set in the logic section of the namelist builder will
       # automatically set these correctly (well that is the assumption), but here we
       # want to set a catch to fail and warn users if they explicitly set incompatible user namelist
@@ -774,7 +774,7 @@ sub setup_cmdl_fates_mode {
        # dis-allow fates specific namelist items with non-fates runs
        my @list  = (  "fates_spitfire_mode", "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys",
                       "use_fates_cohort_age_tracking",
-                      "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp","use_fates_sp","fates_inventory_ctrl_filename","use_fates_logging","fates_parteh_mode" );
+                      "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp","use_fates_sp","fates_inventory_ctrl_filename","use_fates_logging","fates_parteh_mode","use_fates_tree_damage" );
        # dis-allow fates specific namelist items with non-fates runs
        foreach my $var ( @list ) {
           if ( defined($nl->get_value($var)) ) {
@@ -863,16 +863,34 @@ sub setup_cmdl_bgc {
      }
   }
   #
+  # Set FATES-SP mode
+  #
+  if ( &value_is_true( $nl_flags->{'use_fates'} ) ) {
+     add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_sp', 'use_fates'=>$nl_flags->{'use_fates'} );
+     if ( &value_is_true($nl->get_value('use_fates_sp')) ) {
+        $nl_flags->{'use_fates_sp'} = ".true.";
+     } else {
+        $nl_flags->{'use_fates_sp'} = ".false.";
+     }
+  } else {
+     $nl_flags->{'use_fates_sp'} = ".false.";
+  }
+  #
   # Determine Soil decomposition method
   #
   my $var = "soil_decomp_method";
   add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var,
-              'phys'=>$nl_flags->{'phys'}, 'use_cn'=>$nl_flags->{'use_cn'}, 'use_fates'=>$nl_flags->{'use_fates'} );
+              'phys'=>$nl_flags->{'phys'}, 'use_cn'=>$nl_flags->{'use_cn'}, 'use_fates'=>$nl_flags->{'use_fates'},
+              'use_fates_sp'=>$nl_flags->{'use_fates_sp'} );
   my $soil_decomp_method = remove_leading_and_trailing_quotes( $nl->get_value( $var ) );
 
-  if ( &value_is_true($nl_flags->{'use_cn'}) ||  &value_is_true($nl_flags->{'use_fates'}))  {
+  if (      &value_is_true($nl_flags->{'use_cn'}) ) {
+     if ( $soil_decomp_method eq "None" ) {
+        $log->fatal_error("$var must NOT be None if use_cn is on");
+     }
+  } elsif ( &value_is_true($nl_flags->{'use_fates'}) && (not &value_is_true($nl_flags->{'use_fates_sp'}))  )  {
      if ( $soil_decomp_method eq "None" ) {
-        $log->fatal_error("$var must NOT be None if use_cn or use_fates are on");
+        $log->fatal_error("$var must NOT be None if use_fates is on and use_fates_sp is not TRUE");
      }
   } elsif ( $soil_decomp_method ne "None" ) {
      $log->fatal_error("$var must be None if use_cn and use_fates are off");
@@ -901,6 +919,12 @@ sub setup_cmdl_bgc {
            $log->warning("$var normally use_nitrif_denitrif should only be FALSE if FATES is on, it has NOT been validated for being off for BGC mode" );
         }
      }
+     # if MIMICS is on and use_fates = .true. then use_lch4 must = .true.
+     if ( (! &value_is_true($nl_flags->{'use_lch4'})) && &value_is_true($nl_flags->{'use_fates'}) ) {
+        if ( $soil_decomp_method eq "MIMICSWieder2015" ) {
+           $log->warning("If MIMICS is on and use_fates = .true. then use_lch4 must be .true. and currently it's not" );
+        }
+     }
   }
   #
   # Set FUN for BGC
@@ -1143,7 +1167,7 @@ sub setup_cmdl_spinup {
      $log->fatal_error("$var has an invalid value ($val). Valid values are: @valid_values");
   }
   if ( $nl_flags->{'bgc_spinup'} eq "on" && (not &value_is_true( $nl_flags->{'use_cn'} ))  && (not &value_is_true($nl_flags->{'use_fates'})) ) {
-     $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor ED is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'}).");
+     $log->fatal_error("$var can not be '$nl_flags->{'bgc_spinup'}' if neither CN nor FATES is turned on (use_cn=$nl_flags->{'use_cn'}, use_fates=$nl_flags->{'use_fates'}).");
   }
   if ( $nl->get_value("spinup_state") eq 0 && $nl_flags->{'bgc_spinup'} eq "on" ) {
      $log->fatal_error("Namelist spinup_state contradicts the command line option bgc_spinup" );
@@ -2022,6 +2046,10 @@ sub setup_logic_subgrid {
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_lake');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_wetland');
    add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'toosmall_urban');
+
+   if ( &value_is_true($nl_flags->{'use_fates'}) && $nl->get_value('n_dom_pfts') != 0 ) {
+      $log->fatal_error( "FATES and n_dom_pfts can NOT be set at the same time" );
+   }
 }
 
 #-------------------------------------------------------------------------------
@@ -2587,7 +2615,7 @@ sub setup_logic_do_transient_crops {
       # In principle, use_fates should be compatible with
       # do_transient_crops. However, this hasn't been tested, so to be safe,
       # we are not allowing this combination for now.
-      $cannot_be_true = "$var has not been tested with ED, so for now these two options cannot be combined";
+      $cannot_be_true = "$var has not been tested with FATES, so for now these two options cannot be combined";
    }
 
    if ($cannot_be_true) {
@@ -3636,6 +3664,10 @@ sub setup_logic_dry_deposition {
   my ($opts, $nl_flags, $definition, $defaults, $nl) = @_;
 
   if ($opts->{'drydep'} ) {
+    if ( &value_is_true( $nl_flags->{'use_fates'}) && not &value_is_true( $nl_flags->{'use_fates_sp'}) ) {
+       $log->warning("DryDeposition can NOT be on when FATES is also on unless FATES-SP mode is on.\n" .
+                     "   Use the '--no-drydep' option when '-bgc fates' is activated");
+    }
     add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'drydep_list');
     add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'drydep_method');
   } else {
@@ -3681,7 +3713,7 @@ sub setup_logic_megan {
 
   if ($nl_flags->{'megan'} ) {
     if ( &value_is_true( $nl_flags->{'use_fates'} ) ) {
-       $log->fatal_error("MEGAN can NOT be on when ED is also on.\n" .
+       $log->warning("MEGAN can NOT be on when FATES is also on.\n" .
                    "   Use the '-no-megan' option when '-bgc fates' is activated");
     }
     add_default($opts,  $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'megan_specifier');
@@ -4081,15 +4113,9 @@ sub setup_logic_fates {
 
     if (&value_is_true( $nl_flags->{'use_fates'})  ) {
         add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'fates_paramfile', 'phys'=>$nl_flags->{'phys'});
-        add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, 'use_fates_sp', 'use_fates'=>$nl_flags->{'use_fates'} );
-        if ( &value_is_true($nl->get_value('use_fates_sp')) ) {
-           $nl_flags->{'use_fates_sp'} = ".true.";
-        } else {
-           $nl_flags->{'use_fates_sp'} = ".false.";
-        }
         my @list  = (  "fates_spitfire_mode", "use_fates_planthydro", "use_fates_ed_st3", "use_fates_ed_prescribed_phys",
                        "use_fates_inventory_init","use_fates_fixed_biogeog","use_fates_nocomp",
-                       "use_fates_logging","fates_parteh_mode", "use_fates_cohort_age_tracking" );
+                       "use_fates_logging","fates_parteh_mode", "use_fates_cohort_age_tracking","use_fates_tree_damage" );
         foreach my $var ( @list ) {
  	  add_default($opts, $nl_flags->{'inputdata_rootdir'}, $definition, $defaults, $nl, $var, 'use_fates'=>$nl_flags->{'use_fates'},
                       'use_fates_sp'=>$nl_flags->{'use_fates_sp'} );

diff --git a/bld/README b/bld/README
@@ -23,7 +23,7 @@ env_run.xml --- Sample case runtime environment variables, so build-namelist can
 listDefaultNamelist.pl -- List the files needed, for a list of resolutions,
                           to run CLM that are currently NOT on your machine.
                           This file can then be used by 
-                            cime/scripts/Tools/check_input_data
+                            cime/CIME/Tools/check_input_data
                           to retreive them from the inputdata repository.
                           Setting up cases with create_newcase also does
                           this -- but only for the exact configuration

diff --git a/bld/listDefaultNamelist.pl b/bld/listDefaultNamelist.pl
@@ -17,7 +17,7 @@
 #
 # To then get the files from the CESM SVN repository:
 #
-# ../cime/scripts/Tools/check_input_data --data-list-dir . --download
+# ../cime/CIME/Tools/check_input_data --data-list-dir . --download
 #
 #=======================================================================
 
@@ -84,7 +84,7 @@ sub usage {
 
   to then read the resulting clm.input_data_list file and retreive the files
 
-  ../cime/scripts/Tools/check_input_data --data-list-dir . --download
+  ../cime/CIME/Tools/check_input_data --data-list-dir . --download
 
 EOF
 }

diff --git a/bld/namelist_files/namelist_defaults_ctsm.xml b/bld/namelist_files/namelist_defaults_ctsm.xml
@@ -491,7 +491,7 @@ attributes from the config_cache.xml file (with keys converted to upper-case).
 <!-- FATES default parameter file                                       -->
 <!-- ================================================================== -->
 
-<fates_paramfile>lnd/clm2/paramdata/fates_params_api.22.1.0_12pft_c220307.nc</fates_paramfile>
+<fates_paramfile>lnd/clm2/paramdata/fates_params_api.24.0.0_12pft_c220608.nc</fates_paramfile>
 
 <!-- ========================================================================================  -->
 <!-- clm 5.0 BGC nitrogen model                                                                -->
@@ -2528,9 +2528,10 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
 <!-- Defaults for different BGC/decomp modes    -->
 <!-- =========================================  -->
 
-<soil_decomp_method use_cn=".true."    >CENTURYKoven2013</soil_decomp_method>
-<soil_decomp_method use_fates=".true." >CENTURYKoven2013</soil_decomp_method>
-<soil_decomp_method                    >None</soil_decomp_method>
+<soil_decomp_method use_cn=".true."                          >CENTURYKoven2013</soil_decomp_method>
+<soil_decomp_method use_fates=".true."                       >CENTURYKoven2013</soil_decomp_method>
+<soil_decomp_method use_fates=".true." use_fates_sp=".true." >None</soil_decomp_method>
+<soil_decomp_method                                          >None</soil_decomp_method>
 
 <use_lch4            soil_decomp_method="None" >.false.</use_lch4>
 <use_lch4                                      >.true.</use_lch4>
@@ -2540,6 +2541,7 @@ lnd/clm2/surfdata_map/release-clm5.0.30/surfdata_ne0np4.CONUS.ne30x8_hist_78pfts
 <!-- ===== FATES DEFAULTS =========== -->
 <fates_spitfire_mode           use_fates=".true.">0</fates_spitfire_mode>
 <use_fates_planthydro          use_fates=".true.">.false.</use_fates_planthydro>
+<use_fates_tree_damage         use_fates=".true.">.false.</use_fates_tree_damage>
 <use_fates_cohort_age_tracking use_fates=".true.">.false.</use_fates_cohort_age_tracking>
 <use_fates_ed_st3              use_fates=".true.">.false.</use_fates_ed_st3>
 <use_fates_ed_prescribed_phys  use_fates=".true.">.false.</use_fates_ed_prescribed_phys>

diff --git a/bld/namelist_files/namelist_definition_ctsm.xml b/bld/namelist_files/namelist_definition_ctsm.xml
@@ -648,6 +648,12 @@ Switch deciding which nutrient model to use in FATES.
 (Only relevant if FATES is on)
 </entry>
 
+<entry id="use_fates_tree_damage" type="logical" category="physics"
+        group="clm_inparm" valid_values="" value=".false."> 
+Toggle to turn on the tree damage module in FATES
+(Only relevant if FATES is on)
+</entry>
+
 <entry id="fates_spitfire_mode" type="integer" category="physics"
         group="clm_inparm" valid_values="0,1,2,3,4,5" value=".false.">
 Turn on spitfire module to simulate fire by setting fates_spitfire_mode > 0.
@@ -2124,7 +2130,7 @@ Command line arguement for biogeochemistry mode for CLM4.5
         Nitrification/De-nitrification
         Methane model
         Vertically resolved Carbon
-   fates = FATES/ED ecosystem demography model with below ground BGC:
+   fates = FATES Functionally Assembled Terrestrial Ecosystem Simulator (ecosystem demography model with below ground BGC):
 </entry>
 
 <!-- ========================================================================================  -->

diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl
@@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile {
 #
 # Figure out number of tests that will run
 #
-my $ntests = 1844;
+my $ntests = 1846;
 if ( defined($opts{'compare'}) ) {
    $ntests += 1254;
 }
@@ -460,7 +460,7 @@ sub cat_and_create_namelistinfile {
                       "-bgc bgc -use_case 1850-2100_SSP3-7.0_transient -namelist '&a start_ymd=20701029/'",
                       "-bgc fates  -use_case 2000_control -no-megan",
                       "-bgc fates  -use_case 20thC_transient -no-megan",
-                      "-bgc fates  -use_case 1850_control -no-megan -namelist '&a use_fates_sp=T/'",
+                      "-bgc fates  -use_case 1850_control -no-megan -namelist \"&a use_fates_sp=T, soil_decomp_method='None'/\"",
                       "-bgc sp  -use_case 2000_control -res 0.9x1.25 -namelist '&a use_soil_moisture_streams = T/'",
                       "-bgc bgc -use_case 1850-2100_SSP5-8.5_transient -namelist '&a start_ymd=19101023/'",
                       "-bgc bgc -use_case 2000_control -namelist \"&a fire_method='nofire'/\" -crop",
@@ -956,6 +956,11 @@ sub cat_and_create_namelistinfile {
                                      GLC_TWO_WAY_COUPLING=>"FALSE",
                                      phys=>"clm5_0",
                                    },
+     "useFATESWn_dom_pft"        =>{ options=>"-bgc fates -envxml_dir . -no-megan",
+                                     namelst=>"n_dom_pfts = 1",
+                                     GLC_TWO_WAY_COUPLING=>"FALSE",
+                                     phys=>"clm5_0",
+                                   },
      "useFATESWbMH"              =>{ options=>"-bgc fates -envxml_dir . -no-megan",
                                      namelst=>"use_biomass_heat_storage=.true.",
                                      GLC_TWO_WAY_COUPLING=>"FALSE",
@@ -1001,6 +1006,11 @@ sub cat_and_create_namelistinfile {
                                      GLC_TWO_WAY_COUPLING=>"FALSE",
                                      phys=>"clm4_5",
                                    },
+     "useDRYDEPwithFATES"        =>{ options=>"--bgc fates --envxml_dir . --no-megan --drydep",
+                                     namelst=>"",
+                                     GLC_TWO_WAY_COUPLING=>"FALSE",
+                                     phys=>"clm4_5",
+                                   },
      "useFATESSPWONOCOMP"        =>{ options=>"-bgc fates -envxml_dir . -no-megan",
                                      namelst=>"use_fates_sp=T,use_fates_nocomp=F",
                                      GLC_TWO_WAY_COUPLING=>"FALSE",

diff --git a/cime_config/buildlib b/cime_config/buildlib
@@ -9,7 +9,7 @@ _CIMEROOT = os.environ.get("CIMEROOT")
 if _CIMEROOT is None:
     raise SystemExit("ERROR: must set CIMEROOT environment variable")
 
-_LIBDIR = os.path.join(_CIMEROOT, "scripts", "Tools")
+_LIBDIR = os.path.join(_CIMEROOT, "CIME", "Tools")
 sys.path.append(_LIBDIR)
 
 from standard_script_setup import *

diff --git a/cime_config/buildnml b/cime_config/buildnml
@@ -9,7 +9,7 @@ _CIMEROOT = os.environ.get("CIMEROOT")
 if _CIMEROOT is None:
     raise SystemExit("ERROR: must set CIMEROOT environment variable")
 
-_LIBDIR = os.path.join(_CIMEROOT, "scripts", "Tools")
+_LIBDIR = os.path.join(_CIMEROOT, "CIME", "Tools")
 sys.path.append(_LIBDIR)
 
 from standard_script_setup          import *