clm5nl-gen v0.6 (ESCOMP#16)

Commit summary:

* 437789c48 Bumped clm5nl-gen version to 0.6
* b35675172 Wrapped lines based on max column width
* 0b3e1b53c Automatically set user-specified namelist parameters
* bb5caa29f Made reading/writing/checking valid namelist parameters more convenient 
* 13286e75f Fixed __contains__ and __str__ in NamelistGroupMixin
* 7ef720dee Removed <nl>.user_nl and introduced <nl>.general_options

namelist_generator/clm5nl-gen

@@ -54,18 +54,21 @@ def generate_namelists(params_file, out_dir: str = ""):
   print(f"Namelists will be saved to {out_dir}")
   os.makedirs(out_dir, exist_ok=True)   
   for nl, opts in nl_opts.items():
+    if "general_options" not in opts:
+      opts["general_options"] = {}
+
     if nl == "drv_in":
-      if "case_name" not in opts:
-        opts["case_name"] = os.path.splitext(os.path.basename(params_file))[0]
+      if "case_name" not in opts["general_options"]:
+        opts["general_options"]["case_name"] = os.path.splitext(os.path.basename(params_file))[0]
     elif nl == "lnd_in":
-      opts["drv_in.start_type"] = nl_opts["drv_in"]["clm_start_type"]
+      opts["general_options"]["drv_in.start_type"] = nl_opts["drv_in"]["general_options"]["clm_start_type"]
     elif nl == "drv_flds_in":
-      opts["lnd_in.clm_accelerated_spinup"] = nl_opts["lnd_in"]["clm_accelerated_spinup"]
-      opts["lnd_in.use_fates"] = nl_opts["lnd_in"]["use_fates"]
+      opts["general_options"]["lnd_in.clm_accelerated_spinup"] = nl_opts["lnd_in"]["general_options"]["clm_accelerated_spinup"]
+      opts["general_options"]["lnd_in.use_fates"] = nl_opts["lnd_in"]["general_options"]["use_fates"]
     elif nl == "mosart_in":
-      opts["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}")
+      opts["general_options"]["frivinp_rtm"] = resolve_env_vars("${CESMDATAROOT}")
     elif nl == "modelio_nml":
-      opts["drv_in.ntasks"] = nl_opts["drv_in"]["ntasks"]
+      opts["general_options"]["drv_in.ntasks"] = nl_opts["drv_in"]["general_options"]["ntasks"]
     success, msg = build_namelist(nl, opts, out_dir)
     if not success: print(f'ERROR in build_namelist("{nl}"): {msg}', file=sys.stderr) ; return 3
 

namelist_generator/clm5nl/_version.py

@@ -1 +1 @@
-__version__ = "0.5"
+__version__ = "0.6"

namelist_generator/clm5nl/generators/gen_datm_in.py

@@ -20,12 +20,13 @@
 def build_datm_in(opts: dict = None, nl_file: str = "datm_in"):
     global _opts, _user_nl, _nl
 
-    _opts = opts
-    _user_nl = opts.get("user_nl", {})
+    _opts = opts.get("general_options", {})
+    _user_nl = opts.copy()
+    _user_nl.pop("general_options", {})
     _nl = datm_in()
 
     # Validate inputs
-    if _opts["domainfile"] is None:
+    if _user_nl["domainfile"] is None:
         error("datm domainfile must be specified.")
     if _opts["datm_presaero"] is not None and _opts["datm_presaero"] == "none":
         error("datm_presaero = 'none' is not supported.")
@@ -34,6 +35,9 @@ def build_datm_in(opts: dict = None, nl_file: str = "datm_in"):
     shr_strdata_nml()
     datm_nml()
 
+    # Set user-specified namelist parameters
+    _nl.update(_user_nl)
+
     # Write to file
     if nl_file and Path(nl_file).name.strip() != "": 
         _nl.write(nl_file, ["datm_nml", "shr_strdata_nml"])
@@ -63,7 +67,7 @@ def datm_nml():
 def shr_strdata_nml():
     with _nl.shr_strdata_nml as n:
         n.datamode   = "CLMNCEP" 
-        n.domainfile = _opts["domainfile"] # not optional!
+        n.domainfile = _user_nl["domainfile"] # not optional!
         if "streams" in _user_nl:
             n.streams = _user_nl["streams"]
         else:
@@ -122,15 +126,15 @@ def create_stream_files(out_dir : str):
         s_files.append(s_file)
         if s_type == "presaero":
             s_vars = deepcopy(PRESAERO_STREAM_DEFAULTS)
-            s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
-            s_vars["FIELD_FILE_PATH"]   = Path(_opts["domainfile"]).parent.absolute()
+            s_vars["DOMAIN_FILE_PATH"]  = Path(_user_nl["domainfile"]).parent.absolute()
+            s_vars["FIELD_FILE_PATH"]   = Path(_user_nl["domainfile"]).parent.absolute()
         elif s_type == "topo":
             s_vars = deepcopy(TOPO_STREAM_DEFAULTS)
-            s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
-            s_vars["FIELD_FILE_PATH"]   = Path(_opts["domainfile"]).parent.absolute()
+            s_vars["DOMAIN_FILE_PATH"]  = Path(_user_nl["domainfile"]).parent.absolute()
+            s_vars["FIELD_FILE_PATH"]   = Path(_user_nl["domainfile"]).parent.absolute()
         else:
-            s_vars["DOMAIN_FILE_PATH"]  = Path(_opts["domainfile"]).parent.absolute()
-            s_vars["DOMAIN_FILE_NAMES"] = Path(_opts["domainfile"]).name
+            s_vars["DOMAIN_FILE_PATH"]  = Path(_user_nl["domainfile"]).parent.absolute()
+            s_vars["DOMAIN_FILE_NAMES"] = Path(_user_nl["domainfile"]).name
             s_vars["FIELD_FILE_PATH"]   = _opts.get("stream_root_dir", "")
             s_vars["DOMAIN_VAR_NAMES"]  = DATM_STREAM_DEFAULTS["DOMAIN_VAR_NAMES"]
             s_vars["FIELD_VAR_NAMES"]   = DATM_STREAM_DEFAULTS["FIELD_VAR_NAMES"].get(s_type, "")

namelist_generator/clm5nl/generators/gen_drv_flds_in.py

@@ -10,13 +10,14 @@
 
 __all__ = ['build_drv_flds_in']
 _opts = {}
+_user_nl = {}
 _nl = drv_flds_in()
 
 def build_drv_flds_in(opts: dict = None, nl_file: str = "drv_flds_in"):
 
-    global _opts, _nl
-    _opts = opts
-    _user_nl = opts.get("user_nl", {})
+    global _opts, _user_nl, _nl
+    _opts = opts.get("general_options", {})
+    _user_nl = opts
     _nl = drv_flds_in()
 
     _opts["megan"] = opts.get("megan", "default")
@@ -82,8 +83,8 @@ def setup_logic_megan():
                         "CH3CHO = acetaldehyde",
                         "CH3COOH = acetic_acid",
                         "CH3COCH3 = acetone"]
-            if _opts["megan_factors_file"] is not None:
-                n.megan_factors_file = _opts["megan_factors_file"]
+            if _user_nl["megan_factors_file"] is not None:
+                n.megan_factors_file = _user_nl["megan_factors_file"]
             else:
                 n.megan_factors_file = "atm/cam/chem/trop_mozart/emis/megan21_emis_factors_78pft_c20161108.nc"
         else:
@@ -103,6 +104,6 @@ def setup_logic_megan():
     opts["drydep"] = False
     opts["fire_emis"] = False
     opts["megan"] = True
-    opts["megan_factors_file"] = "/p/scratch/nrw_test_case/megan21_emis_factors_78pft_c20161108.nc"
+    user_nl["megan_factors_file"] = "/p/scratch/nrw_test_case/megan21_emis_factors_78pft_c20161108.nc"
 
     build_drv_flds_in(opts, user_nl, "drv_flds_in_test")

namelist_generator/clm5nl/generators/gen_drv_in.py

@@ -24,8 +24,9 @@
 def build_drv_in(opts: dict = None, nl_file: str = "drv_in"):
     global _opts, _user_nl, _nl
 
-    _opts = opts
-    _user_nl = opts.get("user_nl", {})
+    _opts = opts.get("general_options", {})
+    _user_nl = opts.copy()
+    _user_nl.pop("general_options", {})
     _nl = drv_in()
 
     _opts["ATM_NCPL"] = opts.get("ATM_NCPL", 48)
@@ -58,6 +59,9 @@ def build_drv_in(opts: dict = None, nl_file: str = "drv_in"):
     seq_cplflds_userspec()
     seq_flux_mct_inparm()
 
+    # Set user-specified namelist parameters
+    _nl.update(_user_nl)
+
     #Write to file
     if nl_file and Path(nl_file).name.strip() != "":
         _nl.write(nl_file)

namelist_generator/clm5nl/generators/gen_lnd_in.py

@@ -22,10 +22,11 @@
 def build_lnd_in(opts: dict = None, nl_file: str = "lnd_in"):
 
     # Initialize module level variables
-    global _opts, _user_nl
+    global _opts, _user_nl, _nl
     global _nl, _env
-    _opts = opts
-    _user_nl = opts.get("user_nl", {})
+    _opts = opts.get("general_options", {})
+    _user_nl = opts.copy()
+    _user_nl.pop("general_options", {})
     _nl = lnd_in()
 
     # set defaults
@@ -64,7 +65,10 @@ def build_lnd_in(opts: dict = None, nl_file: str = "lnd_in"):
 
     # this param is needed by drv_flds_in
     _opts["use_fates"] = _nl.clm_inparm.use_fates
-
+
+    # Set user-specified namelist parameters
+    _nl.update(_user_nl)
+
     # Write to file
     if nl_file and Path(nl_file).name.strip() != "":
         _nl.write(nl_file, lnd_nl_groups())

namelist_generator/clm5nl/generators/gen_mosart_in.py

@@ -31,7 +31,7 @@ def build_mosart_in(opts: dict = None, nl_file: str = "mosart_in"):
         n.do_rtm = False
         n.do_rtmflood = False
         n.finidat_rtm = " "
-        n.frivinp_rtm = opts["frivinp_rtm"]
+        n.frivinp_rtm = opts["general_options"]["frivinp_rtm"]
         n.ice_runoff = True 
         n.qgwl_runoff_option = "threshold"
         n.rtmhist_fexcl1 = ""

namelist_generator/clm5nl/structures/namelist.py

@@ -1,9 +1,10 @@
 from collections import OrderedDict
-from .utils import nml2str
+from .utils import nml2str, nmlGroup2str
 
 class Namelist(object):
     def __init__(self, nl_obj: dict = None):
         self._name = type(self).__name__
+        self._valid_params = {}
         if nl_obj is None:
             self._nl = OrderedDict()
         else:
@@ -13,14 +14,39 @@ def __iter__(self):
         return self._nl.__iter__()
 
     def __len__(self):
-        acc = 0
-        for v in self._nl.values():
-            acc += len(v)
-        return acc
+        param_count = 0
+        for nl_group in self._nl.values():
+            param_count += len(nl_group)
+        return param_count
+
+    def __getitem__(self, key):
+        if key in self._valid_params:
+            return self._nl[self._valid_params[key]].get(key, None)
+        else:
+            raise KeyError(f"'{key}' is not a valid {self._name} parameter.")
+
+    def __setitem__(self, key, value):
+        if key in self._valid_params:
+            self._nl[self._valid_params[key]][key] = value
+        else:
+            raise KeyError(f"'{key}' is not a valid {self._name} parameter.")
+
+    def __contains__(self, item):
+        return item in self._valid_params.keys()
+
+    def _update_valid_params(self, valid_params: dict):
+        self._valid_params.update(valid_params)
 
     def keys(self):
         return self._nl.keys()
 
+    def update(self, other=None, **kwargs):
+        if other is not None:
+            for k, v in other.items():
+                self[k] = v
+        for k, v in kwargs.items():
+            self[k] = v
+
     def __str__(self):
         return nml2str(self._nl)
 
@@ -29,21 +55,24 @@ def write(self, file_path, grp_names: list = []):
             f.write(nml2str(self._nl, grp_names=grp_names))
 
 class NamelistGroupMixin(object):
-    def __init__(self):
+    def __init__(self, parent: Namelist, valid_params: dict):
         self._group_name = type(self).__name__
-        self._parent: Namelist = None
+        self._parent = parent
+        self._valid_params = valid_params
+        self._parent._update_valid_params(valid_params)
 
-    def _get_param(self, key):
+    def __getitem__(self, key):
         if self._group_name in self._parent._nl:
             return self._parent._nl[self._group_name].get(key, None)
         else:
             return None
 
-    def _set_param(self, key, value):
-        if not self._group_name in self._parent._nl: self._parent._nl[self._group_name] = OrderedDict()
+    def __setitem__(self, key, value):
+        if not self._group_name in self._parent._nl: 
+            self._parent._nl[self._group_name] = OrderedDict()
         self._parent._nl[self._group_name][key] = value
 
-    def _del_param(self, key):
+    def __delitem__(self, key):
         if self._group_name in self._parent._nl:
            del self._parent._nl[self._group_name][key]
 
@@ -60,10 +89,7 @@ def __iter__(self):
             return {}.__iter__()
 
     def __contains__(self, item):
-        if self._group_name in self._parent._nl:
-            return (item in self._parent._nl[self._group_name])
-        else:
-            return False
+        return item in self._valid_params.keys()
 
     def __len__(self):
         if self._group_name in self._parent._nl:
@@ -72,22 +98,17 @@ def __len__(self):
             return 0 
 
     def __str__(self):
-        return nml2str(self._parent._nl[self._group_name], [self._group_name], False)
-
-    def items(self):
-        if self._group_name in self._parent._nl:
-            return self._parent._nl[self._group_name].items()
-        else:
-            return {}.items()
+        return nmlGroup2str(self._group_name, {p:self[p] for p in self._valid_params})
 
 class namelist_group():
     def __init__(self, nl_grp):
         self._group_type = type(nl_grp.__name__, (nl_grp, NamelistGroupMixin), {})
         self._group = None
+        self._params = {p:nl_grp.__name__ for p in dir(nl_grp) if not p.startswith('_')}
 
     def __get__(self, nl_obj, objtype):
-        if self._group is None: self._group = self._group_type()
-        self._group._parent = nl_obj
+        if self._group is None: 
+            self._group = self._group_type(parent = nl_obj, valid_params = self._params)
         return self._group
 
 class namelist_item(object):
@@ -96,10 +117,10 @@ def __init__(self, func):
         self.__doc__ = func.__doc__
 
     def __get__(self, nl_grp, objtype):
-        return nl_grp._get_param(self._param_name)
+        return nl_grp[self._param_name]
 
     def __set__(self, nl_grp, value):
-        nl_grp._set_param(self._param_name, value)
+        nl_grp[self._param_name] = value
 
     def __delete__(self, nl_grp):
-        nl_grp._del_param(self._param_name)
+        del nl_grp[self._param_name]

namelist_generator/clm5nl/structures/utils.py

@@ -2,7 +2,7 @@
 from itertools import filterfalse
 from io import StringIO
 
-NL_COLUMN_WIDTH_MAX = 80
+NL_COLUMN_WIDTH_MAX = 60
 
 def nml2str(nl: dict, grp_names: list = [], fill_missing_groups: bool = True) -> str:
     """
@@ -56,6 +56,8 @@ def py2fortran(obj) -> str:
             value = ", ".join(f"'{s}'" for s in obj)
         else:
             value = ", ".join(str(s) for s in obj)
+    elif obj is None:
+        value = "''"
     else:
         value = str(obj)
     return value
@@ -69,12 +71,18 @@ def wrap(line):
     key = line[:after_eq_sign]
     params = [p.strip() for p in line[after_eq_sign:].split(",")]
     if len(params) > 1:
-        # Key length includes the equal sign and the space after it
+        # Indent length includes the parameter, equal sign, and the space after it
         indent = " " * (len(key) + 1)
-        # Indent parameters based on key length
-        wrapped_lines =  [f"{indent}{p}" for p in params]
-        # Replace indent on the 1st parameter with a leading space
-        wrapped_lines[0] = " " + wrapped_lines[0].strip()
+
+        wrapped_lines = []
+        l = " " + params[0]        # 1st parameter
+        for p in params[1:]:
+            if len(l) < NL_COLUMN_WIDTH_MAX:
+                l += ", " + p
+            else:
+                wrapped_lines.append(l)
+                l = f"{indent}{p}" # start of new line
+        wrapped_lines.append(l)    # last set of parameters
         return key + ",\n".join(wrapped_lines)
     else:
         return line