Modify mesh mask with new mesh_mask_modifier tool

.git-blame-ignore-revs

@@ -2,4 +2,8 @@
 4cd83cb3ee6d85eb909403487abf5eeaf4d98911
 0aa2957c1f8603c63fa30b11295c06cfddff44a5
 2cdb380febb274478e84cd90945aee93f29fa2e6
+e44dc469439e02e9ee582dab274d890ebdfab104
+565e7bcacbb0bba20ff80d680d8ad769676641c6
+b88e1cd1b28e3609684c79a2ec0e88f26cfc362b
 51c102c5df2e0ef971b5f8eeeb477567899af63a
+7dacad70e74e2ec97f6492d4e7a3cb5dd498bcd7

cime_config/SystemTests/fsurdatmodifyctsm.py

@@ -48,7 +48,7 @@ def __init__(self, case):
 
     def _create_config_file(self):
         cfg_template_path = os.path.join(self._ctsm_root,
-            'tools/modify_fsurdat/modify_template.cfg')
+            'tools/modify_input_files/modify_fsurdat_template.cfg')
 
         with open (self._cfg_file_path,'w') as cfg_out:
             with open (cfg_template_path,'r') as cfg_in:
@@ -64,7 +64,7 @@ def _create_config_file(self):
 
     def _run_modify_fsurdat(self):
         tool_path = os.path.join(self._ctsm_root,
-                                 'tools/modify_fsurdat/fsurdat_modifier')
+                                 'tools/modify_input_files/fsurdat_modifier')
         # Need to specify a specific python version that has the required
         # dependencies
         python_path = _get_python_path()

doc/ChangeLog

@@ -1,4 +1,82 @@
 ===============================================================
+Tag name: ctsm5.1.dev110
+Originator(s): slevis (Samuel Levis, 303-665-1310)
+Date: Mon Sep 26 11:44:52 MDT 2022
+One-line Summary: Introduction of modify_meshes tool for use in I-cases and F-cases
+
+Purpose and description of changes
+----------------------------------
+
+ Merging the modify_meshes tool to main. This tool is a follow-up to the
+ modify_fsurdat tool for I- and F-cases where users are adding land to
+ areas that by default were expected as ocean. In such cases, fsurdat
+ modifications will take effect only if users also modify the mask_mesh
+ file which ultimately determines the landmask used by the CTSM.
+ Details in the tool's README.
+
+ Adding test_sys_mesh_modifier.py to python system tests.
+
+ Combining /modify_fsurdat and /modify_mesh_mask directories into one
+ directory: /modify_input_files
+
+
+Significant changes to scientifically-supported configurations
+--------------------------------------------------------------
+
+Does this tag change answers significantly for any of the following physics configurations?
+(Details of any changes will be given in the "Answer changes" section below.)
+
+    [Put an [X] in the box for any configuration with significant answer changes.]
+
+[ ] clm5_1
+
+[ ] clm5_0
+
+[ ] ctsm5_0-nwp
+
+[ ] clm4_5
+
+
+Bugs fixed or introduced
+------------------------
+CTSM issues fixed (include CTSM Issue #): 
+  Fixes #1644 -- Prospective new tool for user to easily modify ocn/lnd mask in their CESM cases
+
+Notes of particular relevance for developers:
+---------------------------------------------
+Changes to tests or testing:
+ Adding test_sys_mesh_modifier.py to python system tests. This requires
+ "module load nco" before running "make stest" or "make all" in the /python
+ directory.
+
+Testing summary:
+----------------
+[Remove any lines that don't apply.]
+
+ [PASS means all tests PASS; OK means tests PASS other than expected fails.]
+
+  python testing (if python code has changed; see instructions in python/README.md; document testing done):
+
+    cheyenne - PASS
+    izumi - mostly PASS if approp. environment is loaded, but one test fails
+            because it requires "module load nco"
+    clm_pymods on cheyenne - PASS
+
+If the tag used for baseline comparisons was NOT the previous tag, note that here:
+
+Answer changes
+--------------
+
+Changes answers relative to baseline: No
+
+Other details
+-------------
+
+Pull Requests that document the changes (include PR ids):
+ https://github.com/ESCOMP/CTSM/pull/1677 -- Modify mesh mask with new mesh_mask_modifier tool
+
+===============================================================
+===============================================================
 Tag name: ctsm5.1.dev109
 Originator(s): slevis (Samuel Levis,SLevis Consulting,303-665-1310)
 Date: Mon Sep 26 10:46:33 MDT 2022

doc/ChangeSum

@@ -1,5 +1,6 @@
 Tag                   Who      Date  Summary
 ============================================================================================================================
+    ctsm5.1.dev110   slevis 09/26/2022 Introduction of modify_meshes tool for use in I-cases and F-cases
     ctsm5.1.dev109   slevis 09/26/2022 If not MIMICS, do not output certain MIMICS history fields
     ctsm5.1.dev108  afoster 09/08/2022 Connect ozone from atmosphere
     ctsm5.1.dev107  afoster 09/07/2022 Update Externals

python/README.md

@@ -5,6 +5,8 @@
 To run all tests (unit tests, system tests and pylint), simply run `make
 all` from this directory.
 
+When you run `make all`, you need to first execute `module load nco`.
+
 ## Python environment
 
 Another way is to use the file conda\_env\_ctsm\_py.txt to setup
@@ -39,11 +41,17 @@ thing, but support different options:
    `make utest` or `make stest`, or they can all be run with `make
    test`.
 
+   When you run `make test` or `make stest`, you need to first execute
+   `module load nco`.
+
 2. via `./run_ctsm_py_tests`
 
    You can specify various arguments to this; run `./run_ctsm_py_tests
    -h` for details
 
+   In any configuration where you run the system tests, you need to
+   first execute `module load nco`.
+
 ## pylint
 
 You can run pylint on everything in the ctsm package with `make lint`.

python/ctsm/modify_fsurdat/fsurdat_modifier.py → python/ctsm/modify_input_files/fsurdat_modifier.py

@@ -1,20 +1,23 @@
 """
 Run this code by using the following wrapper script:
-tools/modify_fsurdat/fsurdat_modifier
+tools/modify_input_files/fsurdat_modifier
 
 The wrapper script includes a full description and instructions.
 """
 
+import os
 import logging
 import argparse
 from configparser import ConfigParser
+
+from ctsm.utils import abort, write_output
 from ctsm.config_utils import get_config_value
 from ctsm.ctsm_logging import (
     setup_logging_pre_config,
     add_logging_args,
     process_logging_args,
 )
-from ctsm.modify_fsurdat.modify_fsurdat import ModifyFsurdat
+from ctsm.modify_input_files.modify_fsurdat import ModifyFsurdat
 
 logger = logging.getLogger(__name__)
 
@@ -107,11 +110,30 @@ def fsurdat_modifier(cfg_path):
         can_be_unset=True,
     )
 
+    lat_dimname = get_config_value(
+        config=config, section=section, item="lat_dimname", file_path=cfg_path, can_be_unset=True
+    )
+    lon_dimname = get_config_value(
+        config=config, section=section, item="lon_dimname", file_path=cfg_path, can_be_unset=True
+    )
+
     # Create ModifyFsurdat object
     modify_fsurdat = ModifyFsurdat.init_from_file(
-        fsurdat_in, lnd_lon_1, lnd_lon_2, lnd_lat_1, lnd_lat_2, landmask_file
+        fsurdat_in,
+        lnd_lon_1,
+        lnd_lon_2,
+        lnd_lat_1,
+        lnd_lat_2,
+        landmask_file,
+        lat_dimname,
+        lon_dimname,
     )
 
+    # If output file exists, abort before starting work
+    if os.path.exists(fsurdat_out):
+        errmsg = "Output file already exists: " + fsurdat_out
+        abort(errmsg)
+
     # not required: user may set these in the .cfg file
     max_pft = int(max(modify_fsurdat.file.lsmpft))
     dom_pft = get_config_value(
@@ -231,4 +253,4 @@ def fsurdat_modifier(cfg_path):
     # ----------------------------------------------
     # Output the now modified CTSM surface data file
     # ----------------------------------------------
-    modify_fsurdat.write_output(fsurdat_in, fsurdat_out)
+    write_output(modify_fsurdat.file, fsurdat_in, fsurdat_out, "fsurdat")

python/ctsm/modify_input_files/mesh_mask_modifier.py

@@ -0,0 +1,90 @@
+"""
+Run this code by using the following wrapper script:
+tools/modify_input_files/mesh_mask_modifier
+
+The wrapper script includes a full description and instructions.
+"""
+
+import os
+import logging
+import argparse
+from configparser import ConfigParser
+
+from ctsm.utils import abort, write_output
+from ctsm.config_utils import get_config_value
+from ctsm.ctsm_logging import setup_logging_pre_config, add_logging_args, process_logging_args
+from ctsm.modify_input_files.modify_mesh_mask import ModifyMeshMask
+
+logger = logging.getLogger(__name__)
+
+
+def main():
+    """
+    Description
+    -----------
+    Calls function that modifies mesh mask
+    """
+
+    # set up logging allowing user control
+    setup_logging_pre_config()
+
+    # read the command line argument to obtain the path to the .cfg file
+    parser = argparse.ArgumentParser()
+    parser.add_argument("cfg_path", help="/path/name.cfg of input file, eg ./modify.cfg")
+    add_logging_args(parser)
+    args = parser.parse_args()
+    process_logging_args(args)
+    mesh_mask_modifier(args.cfg_path)
+
+
+def mesh_mask_modifier(cfg_path):
+    """Implementation of mesh_mask_modifier command"""
+    # read the .cfg (config) file
+    config = ConfigParser()
+    config.read(cfg_path)
+    section = config.sections()[0]  # name of the first section
+
+    # required: user must set these in the .cfg file
+    mesh_mask_in = get_config_value(
+        config=config, section=section, item="mesh_mask_in", file_path=cfg_path
+    )
+    mesh_mask_out = get_config_value(
+        config=config, section=section, item="mesh_mask_out", file_path=cfg_path
+    )
+    landmask_file = get_config_value(
+        config=config, section=section, item="landmask_file", file_path=cfg_path
+    )
+    lat_dimname = get_config_value(
+        config=config, section=section, item="lat_dimname", file_path=cfg_path
+    )
+    lon_dimname = get_config_value(
+        config=config, section=section, item="lon_dimname", file_path=cfg_path
+    )
+    lat_varname = get_config_value(
+        config=config, section=section, item="lat_varname", file_path=cfg_path
+    )
+    lon_varname = get_config_value(
+        config=config, section=section, item="lon_varname", file_path=cfg_path
+    )
+
+    # Create ModifyMeshMask object
+    modify_mesh_mask = ModifyMeshMask.init_from_file(
+        mesh_mask_in, landmask_file, lat_dimname, lon_dimname, lat_varname, lon_varname
+    )
+
+    # If output file exists, abort before starting work
+    if os.path.exists(mesh_mask_out):
+        errmsg = "Output file already exists: " + mesh_mask_out
+        abort(errmsg)
+
+    # ----------------
+    # modify mesh mask
+    # ----------------
+
+    # Modify mesh mask
+    modify_mesh_mask.set_mesh_mask("elementMask")
+
+    # ----------------------------------------------
+    # Output the now modified CTSM surface data file
+    # ----------------------------------------------
+    write_output(modify_mesh_mask.file, mesh_mask_in, mesh_mask_out, "mesh")

python/ctsm/modify_fsurdat/modify_fsurdat.py → python/ctsm/modify_input_files/modify_fsurdat.py

@@ -1,18 +1,19 @@
 """
 Run this code by using the following wrapper script:
-/tools/modify_fsurdat/fsurdat_modifier
+/tools/modify_input_files/fsurdat_modifier
 
 The wrapper script includes a full description and instructions.
 """
 
 import os
 import logging
 
+from math import isclose
 import numpy as np
 import xarray as xr
 
-from ctsm.git_utils import get_ctsm_git_short_hash
 from ctsm.utils import abort, update_metadata
+from ctsm.git_utils import get_ctsm_git_short_hash
 from ctsm.config_utils import lon_range_0_to_360
 
 logger = logging.getLogger(__name__)
@@ -24,7 +25,9 @@ class ModifyFsurdat:
     -----------
     """
 
-    def __init__(self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file):
+    def __init__(
+        self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file, lat_dimname, lon_dimname
+    ):
 
         self.file = my_data
 
@@ -40,17 +43,36 @@ def __init__(self, my_data, lon_1, lon_2, lat_1, lat_2, landmask_file):
         if landmask_file is not None:
             # overwrite self.not_rectangle with data from
             # user-specified .nc file in the .cfg file
-            self._landmask_file = xr.open_dataset(landmask_file)
-            self.rectangle = self._landmask_file.landmask
+            landmask_ds = xr.open_dataset(landmask_file)
+            self.rectangle = landmask_ds.mod_lnd_props.data
+            # CF convention has dimension and coordinate variable names the same
+            if lat_dimname is None:  # set to default
+                lat_dimname = "lsmlat"
+            if lon_dimname is None:  # set to default
+                lon_dimname = "lsmlon"
+            lsmlat = landmask_ds.dims[lat_dimname]
+            lsmlon = landmask_ds.dims[lon_dimname]
+
+            for row in range(lsmlat):  # rows from landmask file
+                for col in range(lsmlon):  # cols from landmask file
+                    errmsg = (
+                        "landmask_ds.mod_lnd_props not 0 or 1 at "
+                        + f"row, col, value = {row} {col} {self.rectangle[row, col]}"
+                    )
+                    assert isclose(self.rectangle[row, col], 0, abs_tol=1e-9) or isclose(
+                        self.rectangle[row, col], 1, abs_tol=1e-9
+                    ), errmsg
 
         self.not_rectangle = np.logical_not(self.rectangle)
 
     @classmethod
-    def init_from_file(cls, fsurdat_in, lon_1, lon_2, lat_1, lat_2, landmask_file):
+    def init_from_file(
+        cls, fsurdat_in, lon_1, lon_2, lat_1, lat_2, landmask_file, lat_dimname, lon_dimname
+    ):
         """Initialize a ModifyFsurdat object from file fsurdat_in"""
         logger.info("Opening fsurdat_in file to be modified: %s", fsurdat_in)
         my_file = xr.open_dataset(fsurdat_in)
-        return cls(my_file, lon_1, lon_2, lat_1, lat_2, landmask_file)
+        return cls(my_file, lon_1, lon_2, lat_1, lat_2, landmask_file, lat_dimname, lon_dimname)
 
     @staticmethod
     def _get_rectangle(lon_1, lon_2, lat_1, lat_2, longxy, latixy):