Merge pull request #3 from NOAA-EMC/develop

Sync HAFS for with recent work in NOAA/EMC

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,12 +8,12 @@ CMEPS Issues Fixed (include github issue #):
 
 Are there dependencies on other component PRs
  - [ ] CIME (list)
- - [ ] CMEPS (list) 
+ - [ ] CMEPS (list)
 
 Are changes expected to change answers?
  - [ ] bit for bit
  - [ ] different at roundoff level
- - [ ] more substantial 
+ - [ ] more substantial
 
 Any User Interface Changes (namelist or namelist defaults changes)?
  - [ ] Yes
@@ -30,13 +30,13 @@ Testing performed:
 Hashes used for testing:
 - [ ] CIME
   - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch: nuopc_dev
+  - branch:
   - hash:
 - [ ] CMEPS
   - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch: nuopc_dev
+  - branch:
   - hash:
 - [ ] CESM
   - repository to check out: https://github.com/ESCOMP/CESM.git
-  - branch: nuopc_dev
+  - branch:
   - hash:

.github/workflows/bumpversion.yml

@@ -0,0 +1,19 @@
+name: Bump version
+on:
+  push:
+    branches:
+      - master
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Bump version and push tag
+        id: tag_version
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          create_annotated_tag: true
+          default_bump: patch
+          dry_run: false
+          tag_prefix: cdeps

.github/workflows/extbuild.yml

@@ -18,10 +18,12 @@ jobs:
       CXX: mpicxx
       CPPFLAGS: "-I/usr/include -I/usr/local/include"
       # Versions of all dependencies can be updated here
-      ESMF_VERSION: ESMF_8_1_0_beta_snapshot_25
-      PNETCDF_VERSION: pnetcdf-1.12.1
+      ESMF_VERSION: ESMF_8_1_1
+      PNETCDF_VERSION: pnetcdf-1.12.2
       NETCDF_FORTRAN_VERSION: v4.5.2
-      PIO_VERSION: pio-2.5.1
+      # These should match in number
+      PIO_VERSION: pio-2.5.4
+      PIO_VERSION_DUMB: pio2_5_4
     steps:
       - uses: actions/checkout@v2
         # Build the ESMF library, if the cache contains a previous build
@@ -32,7 +34,7 @@ jobs:
           path: ~/ESMF
           key: ${{ runner.os }}-${{ env.ESMF_VERSION }}-ESMF
       - id: load-env
-        run: sudo apt-get install gfortran wget openmpi-bin netcdf-bin libopenmpi-dev
+        run: sudo apt-get install gfortran wget openmpi-bin netcdf-bin libopenmpi-dev libnetcdf-dev
       - id: build-ESMF
         if: steps.cache-esmf.outputs.cache-hit != 'true'
         run: |
@@ -93,11 +95,11 @@ jobs:
       - name: Build PIO
         if: steps.cache-PIO.outputs.cache-hit != 'true'
         run: |
-          wget https://github.com/NCAR/ParallelIO/releases/download/pio_2_5_1/${{ env.PIO_VERSION }}.tar.gz
+          wget https://github.com/NCAR/ParallelIO/releases/download/${{ env.PIO_VERSION_DUMB }}/${{ env.PIO_VERSION }}.tar.gz
           tar -xzvf ${{ env.PIO_VERSION }}.tar.gz
           mkdir build-pio
           pushd build-pio
-          cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=On -DPIO_USE_MALLOC=On -DPIO_ENABLE_LOGGING=On -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../${{ env.PIO_VERSION }}
+          cmake -Wno-dev -DNetCDF_C_LIBRARY=/usr/lib/x86_64-linux-gnu/libnetcdf.so -DNetCDF_C_INCLUDE_DIR=/usr/include -DCMAKE_PREFIX_PATH=/usr -DCMAKE_INSTALL_PREFIX=$HOME/pio -DPIO_HDF5_LOGGING=Off -DPIO_ENABLE_EXAMPLES=Off -DPIO_ENABLE_LOGGING=Off -DPIO_ENABLE_TIMING=Off -DNetCDF_Fortran_PATH=$HOME/netcdf-fortran -DPnetCDF_PATH=$HOME/pnetcdf ../${{ env.PIO_VERSION }}
           make VERBOSE=1
           make install
           popd

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "fox"]
-	path = fox
-	url = https://github.com/ESMCI/fox.git

CMakeLists.txt

@@ -1,5 +1,6 @@
 cmake_minimum_required(VERSION 3.10)
 include(ExternalProject)
+include(FetchContent)
 
 if (DEFINED CIMEROOT)
   message("Using CIME in ${CIMEROOT} with compiler ${COMPILER}")
@@ -16,7 +17,8 @@ if (DEFINED CIMEROOT)
     set(CMAKE_Fortran_COMPILER ${MPIFC})
     set(CMAKE_CXX_COMPILER ${MPICXX})
   endif()
-  set(CMAKE_Fortran_FLAGS "${FFLAGS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include")
+  set(CMAKE_Fortran_FLAGS "${FFLAGS} ${CPPDEFS} -I${LIBROOT}/include -I${LIBROOT}/finclude -I${LIBROOT}/nuopc/esmf/${NINST_VALUE}/include")
+  add_compile_definitions(CESMCOUPLED)
 else()
   set(BLD_STANDALONE TRUE)
 endif()
@@ -53,11 +55,31 @@ if(BLD_STANDALONE)
   list(APPEND EXTRA_LIBS cdeps_share)
   list(APPEND EXTRA_INCLUDES "${CMAKE_BINARY_DIR}/share" )
 endif()
-
-add_subdirectory(fox)
+if("${COMPILER}" STREQUAL "nag")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -D__NAG__")
+endif()
 add_subdirectory(streams)
 add_subdirectory(dshr)
 
+if(IS_DIRECTORY "${PROJECT_SOURCE_DIR}/fox")
+  message(STATUS "FoX library is already checked out!")
+  message(STATUS "FoX source dir: ${PROJECT_SOURCE_DIR}/fox")
+else()
+  FetchContent_Declare(fox
+    GIT_REPOSITORY https://github.com/ESMCI/fox.git
+    GIT_TAG 4.1.2.1
+    SOURCE_DIR ${PROJECT_SOURCE_DIR}/fox
+    BINARY_DIR ${PROJECT_SOURCE_DIR}
+  )
+  FetchContent_GetProperties(fox)
+  if(NOT fox_POPULATED)
+    FetchContent_Populate(fox)
+    message(STATUS "FoX source dir: ${fox_SOURCE_DIR}")
+    message(STATUS "FoX binary dir: ${fox_BINARY_DIR}")
+  endif()
+endif()
+add_subdirectory(fox)
+
 foreach(COMP datm dice dlnd docn drof dwav)
   add_subdirectory("${COMP}")
 endforeach(COMP)
@@ -81,22 +103,24 @@ foreach(COMP datm dice dlnd docn drof dwav)
   target_include_directories(${COMP} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/dshr>
                                             $<INSTALL_INTERFACE:mod>)
 
-  install(TARGETS ${COMP} 
+  install(TARGETS ${COMP}
           EXPORT  ${COMP}
+          ARCHIVE DESTINATION lib
           LIBRARY DESTINATION lib
           COMPONENT Library)
-  install(EXPORT      ${COMP} 
+  install(EXPORT      ${COMP}
           DESTINATION lib/cmake)
 endforeach(COMP)
 
 foreach(DEPS streams dshr cdeps_share FoX_dom FoX_wxml FoX_sax FoX_common FoX_utils FoX_fsys)
   if(NOT BLD_STANDALONE AND ${DEPS} STREQUAL "cdeps_share")
      continue()
   endif()
-  install(TARGETS ${DEPS} 
+  install(TARGETS ${DEPS}
           EXPORT  ${DEPS}
+          ARCHIVE DESTINATION lib
           LIBRARY DESTINATION lib
           COMPONENT Library)
-  install(EXPORT      ${DEPS} 
+  install(EXPORT      ${DEPS}
           DESTINATION lib/cmake)
 endforeach(DEPS)

Externals_CDEPS.cfg

@@ -0,0 +1,16 @@
+[fox]
+tag = 4.1.2.1
+protocol = git
+repo_url = https://github.com/ESMCI/fox.git
+local_path = fox
+required = True
+
+[genf90]
+tag = 4816965
+protocol = git
+repo_url = https://github.com/PARALLELIO/genf90
+local_path = share/genf90
+required = True
+
+[externals_description]
+schema_version = 1.0.0

README.md

@@ -1,2 +1,14 @@
 # CDEPS
 Community Data Models for Earth Prediction Systems
+
+For documentation see
+
+https://escomp.github.io/CDEPS/html/index.html
+
+## A note on github tag action
+
+This repository is setup to automatically create tags on merge to
+master using .github/workflows/bumpversion.yml It uses
+https://github.com/mathieudutour/github-tag-action  to look for
+keywords in commit messages and determine what the new version should
+be.  The default if no keywords is found is to bump the patch version.

cime_config/buildlib

@@ -89,7 +89,7 @@ def buildlib(bldroot, libroot, case, compname=None):
             expect(time_counter <= time_to_wait," Timeout waiting for {}".format(dwav_lib))
         print("HERE exeroot {} bldroot {}".format(exe_root, bldroot))
 
-        s,o,e = run_cmd("make install DESTDIR={}".format(libroot), from_dir=bldroot, verbose=True)
+        s,o,e = run_cmd("make install VERBOSE=1 DESTDIR={}".format(libroot), from_dir=bldroot, verbose=True)
         expect(not s,"ERROR from make output={}, error={}".format(o,e))
 
         # Link the CDEPS component directories to the location expected by cime

cime_config/stream_cdeps.py

@@ -42,7 +42,7 @@
  </stream_info>
 
 """
-        
+
 class StreamCDEPS(GenericXML):
 
     def __init__(self, infile, schema):
@@ -64,8 +64,16 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file
             stream_file.write('<file id="stream" version="2.0">\n')
         # write contents of stream file
         for stream_name in stream_names:
-            if stream_name:
-                self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : stream_name}, 
+            if stream_name.startswith("NEON."):
+                self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "NEON.$NEONSITE"},
+                                                                     err_msg="No stream_entry {} found".format(stream_name))
+
+            elif stream_name.startswith("CLM_USRDAT."):
+                self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : "CLM_USRDAT.$CLM_USRDAT_NAME"},
+                                                                     err_msg="No stream_entry {} found".format(stream_name))
+
+            elif stream_name:
+                self.stream_nodes = super(StreamCDEPS,self).get_child("stream_entry", {"name" : stream_name},
                                                                      err_msg="No stream_entry {} found".format(stream_name))
 
             # determine stream_year_first and stream_year_list
@@ -99,46 +107,48 @@ def create_stream_xml(self, stream_names, case, streams_xml_file, data_list_file
                         stream_datafiles = child.xml_element.text
                         stream_datafiles = self._resolve_values(case, stream_datafiles)
                         if 'first_year' in child.xml_element.attrib and 'last_year' in child.xml_element.attrib:
-                            value = child.xml_element.get('first_year') 
+                            value = child.xml_element.get('first_year')
                             value = self._resolve_values(case, value)
                             stream_year_first= int(value)
-                            value = child.xml_element.get('last_year') 
+                            value = child.xml_element.get('last_year')
                             value = self._resolve_values(case, value)
                             stream_year_last = int(value)
                             year_first = max(stream_year_first, data_year_first)
                             year_last = min(stream_year_last, data_year_last)
                             stream_datafiles = self._sub_paths(stream_datafiles, year_first, year_last)
                             stream_datafiles = stream_datafiles.strip()
                         #endif
-                        if stream_vars[node_name]: 
+                        if stream_vars[node_name]:
                             stream_vars[node_name] += "\n      " + self._add_xml_delimiter(stream_datafiles.split("\n"), "file")
                         else:
                             stream_vars[node_name] = self._add_xml_delimiter(stream_datafiles.split("\n"), "file")
 
                 elif (   node_name == 'stream_meshfile'
-                      or node_name == 'stream_mapalgo' 
-                      or node_name == 'stream_tintalgo' 
-                      or node_name == 'stream_taxmode' 
+                      or node_name == 'stream_mapalgo'
+                      or node_name == 'stream_tintalgo'
+                      or node_name == 'stream_taxmode'
                       or node_name == 'stream_dtlimit'):
                     attributes['model_grid'] = case.get_value("GRID")
                     attributes['compset'] = case.get_value("COMPSET")
                     value = self._get_value_match(node, node_name[7:], attributes=attributes)
-                    value = self._resolve_values(case, value)
-                    value = value.strip()
+                    if value:
+                        value = self._resolve_values(case, value)
+                        value = value.strip()
                     stream_vars[node_name] = value
 
                 elif node_name.strip():
                     # Get the other dependencies
                     stream_dict = self._add_value_to_dict(stream_vars, case, node)
-                        
+
             # append to stream xml file
             stream_file_text = _stream_nuopc_file_template.format(**stream_vars)
             with open(streams_xml_file, 'a') as stream_file:
                 stream_file.write(stream_file_text)
 
             # append to input_data_list
-            stream_meshfile = stream_vars['stream_meshfile'].strip()
-            self._add_entries_to_inputdata_list(stream_meshfile, stream_datafiles, data_list_file)
+            if stream_vars['stream_meshfile']:
+                stream_meshfile = stream_vars['stream_meshfile'].strip()
+                self._add_entries_to_inputdata_list(stream_meshfile, stream_datafiles.split("\n"), data_list_file)
 
         # write close of stream xml file
         with open(streams_xml_file, 'a') as stream_file:
@@ -170,7 +180,7 @@ def _add_entries_to_inputdata_list(self, stream_meshfile, stream_datafiles, data
             if hashValue not in lines_hash:
                 input_data_list.write(string)
             # now append the stream_datafile entries
-            for i, filename in enumerate(stream_datafiles.split("\n")):
+            for i, filename in enumerate(stream_datafiles):
                 if filename.strip() == '':
                     continue
                 string = "file{:d} = {}\n".format(i+1, filename)
@@ -193,7 +203,7 @@ def _get_input_file_hash(self, data_list_file):
 
     def _get_value_match(self, node, child_name, attributes=None, exact_match=False):
         '''
-        Get the first best match for multiple tags in child_name based on the 
+        Get the first best match for multiple tags in child_name based on the
         attributes input
 
         <values...>
@@ -397,4 +407,16 @@ def _add_xml_delimiter(list_to_deliminate, delimiter):
         #endfor
         return "\n      ".join(list_to_deliminate)
 
-
+    def update_input_data_list(self, data_list_file):
+        ''' From the stream object parse out and list required input files '''
+        sinodes = self.scan_children("stream_info")
+        for node in sinodes:
+            meshnode = self.scan_child("stream_mesh_file", root=node)
+            stream_meshfile = self.text(meshnode)
+            data_file_node = self.scan_child("stream_data_files", root=node)
+            filenodes = self.scan_children("file",root=data_file_node)
+            stream_datafiles = []
+            for fnode in filenodes:
+                stream_datafiles.append(self.text(fnode))
+            print("HERE {}".format(data_list_file))
+            self._add_entries_to_inputdata_list(stream_meshfile, stream_datafiles, data_list_file)