Update main_v11.0-ref after #2443, #2454, and #2455 (#2456)

Co-authored-by: MET Tools Test Account <[email protected]>
Co-authored-by: Howard Soh <[email protected]>
Co-authored-by: jprestop <[email protected]>
Co-authored-by: hsoh-u <[email protected]>
Co-authored-by: George McCabe <[email protected]>
fix #2389 main_v11.0 flowchart (#2391)
fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2405)
fix #2380 main_v11.0 override (#2381)
fix #2408 main_v11.0 empty config (#2409)
fix #2390 main_v11.0 fix compiling hdf5 with zlib and handle NetCDF-C zip (#2403)
fix #2415 main_v11.0 modulefiles (#2416)
fix #2412 main_v11.0 climo (#2420)
fix #2426 main_v11.0 buoy (#2432)
fix #2437 main_v11.0 convert (#2438)
fix for main_v11.0, for #2437, forgot one reference to the search_parent for a dictionary lookup.
fix 2428 python from env main v11.0 (#2443)
fix 2428 python csv input (#2450)
fix #2452 main_v11.0 airnow (#2453)
fix #2402 main_v11.0 sonarqube (First PR) (#2447)

.readthedocs.yaml

@@ -7,7 +7,7 @@ version: 2
 
 # Build all formats (htmlzip, pdf, epub)
 #formats: all
-formats: [pdf]
+#formats: [pdf]
 
 # Optionally set the version of Python and requirements required to build your
 # docs

data/wrappers/Makefile.am

@@ -24,8 +24,10 @@ wrappers_DATA = \
      set_python_env.py \
      read_tmp_dataplane.py \
      read_tmp_ascii.py \
+     read_tmp_point_nc.py \
      write_tmp_dataplane.py \
      write_tmp_point.py \
+     write_tmp_point_nc.py \
      write_tmp_mpr.py
 
 EXTRA_DIST = ${wrappers_DATA}

data/wrappers/Makefile.in

@@ -359,8 +359,10 @@ wrappers_DATA = \
      set_python_env.py \
      read_tmp_dataplane.py \
      read_tmp_ascii.py \
+     read_tmp_point_nc.py \
      write_tmp_dataplane.py \
      write_tmp_point.py \
+     write_tmp_point_nc.py \
      write_tmp_mpr.py
 
 EXTRA_DIST = ${wrappers_DATA}

data/wrappers/read_tmp_point_nc.py

@@ -0,0 +1,30 @@
+########################################################################
+#
+#    Reads temporary file into memory.
+#
+#    usage:  /path/to/python read_tmp_dataplane.py dataplane.tmp
+#
+########################################################################
+
+import os
+import sys
+
+met_base_dir = os.getenv('MET_BASE',None)
+if met_base_dir is not None:
+    sys.path.append(os.path.join(met_base_dir, 'python'))
+
+# add share/met/python directory to system path to find met_point_obs
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                             os.pardir, 'python')))
+from met_point_obs import met_point_obs
+from met_point_obs_nc import nc_point_obs
+
+netcdf_filename = sys.argv[1]
+
+# read NetCDF file
+print('{p}  reading{f}'.format(p=met_point_obs.get_prompt(), f=netcdf_filename))
+point_obs_data = nc_point_obs()
+point_obs_data.read_data(netcdf_filename)
+
+met_point_data = point_obs_data.get_point_data() 
+met_point_data['met_point_data'] = point_obs_data

data/wrappers/write_tmp_point_nc.py

@@ -0,0 +1,59 @@
+########################################################################
+#
+#    Adapted from a script provided by George McCabe
+#    Adapted by Randy Bullock
+#
+#    usage:  /path/to/python write_tmp_point.py \
+#            tmp_output_filename <user_python_script>.py <args>
+#
+########################################################################
+
+import os
+import sys
+import importlib.util
+
+met_base_dir = os.getenv('MET_BASE',None)
+if met_base_dir is not None:
+    sys.path.append(os.path.join(met_base_dir, 'python'))
+
+# add share/met/python directory to system path to find met_point_obs
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                             os.pardir, 'python')))
+
+from met_point_obs import met_point_obs
+from met_point_obs_nc import nc_point_obs
+
+PROMPT = met_point_obs.get_prompt()
+print("{p} Python Script:\t".format(p=PROMPT)  + repr(sys.argv[0]))
+print("{p} User Command:\t".format(p=PROMPT)   + repr(' '.join(sys.argv[2:])))
+print("{p} Temporary File:\t".format(p=PROMPT) + repr(sys.argv[1]))
+
+tmp_filename = sys.argv[1]
+pyembed_module_name = sys.argv[2]
+sys.argv = sys.argv[2:]
+
+# append user script dir to system path
+pyembed_dir, pyembed_file = os.path.split(pyembed_module_name)
+if pyembed_dir:
+    sys.path.insert(0, pyembed_dir)
+
+if not pyembed_module_name.endswith('.py'):
+    pyembed_module_name += '.py'
+
+user_base = os.path.basename(pyembed_module_name).replace('.py','')
+
+spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name)
+met_in = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(met_in)
+
+if hasattr(met_in, 'point_obs_data'):
+    met_in.point_obs_data.save_ncfile(tmp_filename)
+else:
+    if hasattr(met_in.met_point_data, 'point_obs_data'):
+        met_in.met_point_data['point_obs_data'].save_ncfile(tmp_filename)
+    else:
+        tmp_point_obs = nc_point_obs()
+        tmp_point_obs.put_data(met_in.met_point_data)
+        tmp_point_obs.save_ncfile(tmp_filename)
+
+#print('{p} writing {f}'.format(p=PROMPT, f=tmp_filename))

docs/Users_Guide/release-notes.rst

@@ -9,6 +9,26 @@ When applicable, release notes are followed by the GitHub issue number which des
 enhancement, or new feature (`MET GitHub issues <https://github.com/dtcenter/MET/issues>`_).
 Important issues are listed **in bold** for emphasis.
 
+MET Version 11.0.1-rc1 release notes (20230221)
+-----------------------------------------------
+
+* Bugfixes:
+
+   * Bugfix: Fix the MET CF-Compliant NetCDF library code to Polar Stereographic data from NSIDC Sea Ice Edge NetCDF files (`#2218 <https://github.com/dtcenter/MET/issues/2218>`_).
+   * Bugfix: Remove override keyword to avoid C++11 dependency (`#2380 <https://github.com/dtcenter/MET/issues/2380>`_).
+   * Bugfix: Fix ASCII2NC to not compute AOD 550 if other inputs are negative values (`#2383 <https://github.com/dtcenter/MET/issues/2383>`_).
+   * Bugfix: Fix PB2NC to report accurate total observation counts in log messages (`#2387 <https://github.com/dtcenter/MET/issues/2387>`_).
+   * Bugfix: Update the MET flowchart for version 11.0.0 (`#2389 <https://github.com/dtcenter/MET/issues/2389>`_).
+   * Bugfix: Fix issues with the met_compile_all.sh script and associated tar files (`#2390 <https://github.com/dtcenter/MET/issues/2390>`_).
+   * Bugfix: Correct definitions of NCEP grid numbers 172 and 220 (`#2399 <https://github.com/dtcenter/MET/issues/2399>`_).
+   * Bugfix: Address MET-11.0.0 SonarQube Blocker Bugs (`#2402 <https://github.com/dtcenter/MET/issues/2402>`_).
+   * Bugfix: Refine fix for handling empty configuration files (`#2408 <https://github.com/dtcenter/MET/issues/2408>`_).
+   * Bugfix: Fix time interpolation of monthly climatology data between December 15 and January 15 (`#2412 <https://github.com/dtcenter/MET/issues/2412>`_).
+   * Bugfix: Fix ASCII2NC to handle missing NDBC buoy location information (`#2426 <https://github.com/dtcenter/MET/issues/2426>`_).
+   * Bugfix: Fix the MET vx_pointdata_python library to handle MET_PYTHON_EXE for python embedding of point observations (`#2428 <https://github.com/dtcenter/MET/issues/2428>`_).
+   * Bugfix: Refine the regrid dictionary's data conversion and censoring operations and fix climo time matching logic for a single monthly climo file (`#2437 <https://github.com/dtcenter/MET/issues/2437>`_).
+   * Bugfix: Fix inconsistent ASCII2NC AIRNOW location lookup logic (`#2452 <https://github.com/dtcenter/MET/issues/2452>`_).  
+
 MET Version 11.0.0 release notes (20221209)
 -------------------------------------------
 

docs/conf.py

@@ -20,11 +20,11 @@
 project = 'MET'
 author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES'
 author_list = 'Opatz, J., T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway, K. Newman'
-version = '11.0.0'
+version = '11.0.1-rc1'
 verinfo = version
 release = f'{version}'
-release_year = '2022'
-release_date = f'{release_year}-12-09'
+release_year = '2023'
+release_date = f'{release_year}-02-21'
 copyright = f'{release_year}, {author}'
 
 # -- General configuration ---------------------------------------------------

internal/scripts/installation/compile_MET_all.sh

@@ -832,9 +832,9 @@ if [ $COMPILE_MET -eq 1 ]; then
   # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
   # ${parameter:+word}
   # If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted.
-  export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}"
+  export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${ADDTL_DIR:+:$ADDTL_DIR}${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}"
   export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_JASPER:+$LIB_JASPER}${LIB_LIBPNG:+:$LIB_PNG}${LIB_Z:+$LIB_Z}"
-  export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib}"
+  export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib} ${ADDTL_DIR:+-L$ADDTL_DIR}"
   export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz"
   export MET_FONT_DIR=${TEST_BASE}/fonts
 

internal/scripts/installation/config/install_met_env.acorn_py3.10

@@ -0,0 +1,36 @@
+module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/
+module load intel python/3.10.4
+module load ve/evs/1.0
+module load netcdf/4.7.4
+module load hdf5/1.12.2
+module load bufr/11.5.0
+module load zlib/1.2.11
+module load jasper/2.0.25
+module load libpng/1.6.37
+module load gsl/2.7
+module load g2c/1.6.4
+
+export TEST_BASE=/apps/sw_review/emc/MET/11.0.1
+export LIB_DIR=${TEST_BASE}/external_libs
+export BIN_DIR_PATH=${TEST_BASE}/exec
+export COMPILER=intel_19.1.3.304
+export MET_SUBDIR=${TEST_BASE}
+export MET_TARBALL=v11.0.1.tar.gz
+export USE_MODULES=TRUE
+export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64
+export PYTHON_MODULE=python_3.10.4
+export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp
+export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10
+export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib/python3.10/config-3.10-x86_64-linux-gnu/\ -L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib64\ -lpython3.10\ -lintl\ -lcrypt\ -ldl\ -lutil\ -lm\ -lm
+export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4
+export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2
+export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.5.0/lib64
+export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64
+export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include
+export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb
+export BUFRLIB_NAME=-lbufr_4
+export GRIB2CLIB_NAME=-lg2c
+export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64
+export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib
+export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib
+export SET_D64BIT=FALSE

internal/scripts/installation/config/install_met_env.wcoss2_py3.10

@@ -0,0 +1,47 @@
+# JY module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/
+module load intel
+# JY add following two
+module load craype/2.7.13
+module load cray-mpich/8.1.12
+
+module load python/3.10.4
+module load netcdf/4.7.4
+module load hdf5/1.10.6
+module load bufr/11.6.0
+module load zlib/1.2.11
+module load jasper/2.0.25
+module load libpng/1.6.37
+module load gsl/2.7
+module load g2c/1.6.4
+
+#export TEST_BASE=/apps/ops/para/libs/intel/19.1.3.304/met/11.0.1
+export TEST_BASE=$(pwd)
+export LIB_DIR=${TEST_BASE}/external_libs
+export BIN_DIR_PATH=${TEST_BASE}/bin
+export COMPILER=intel_19.1.3.304
+export MET_SUBDIR=${TEST_BASE}
+export MET_TARBALL=v11.0.1.tar.gz
+export USE_MODULES=TRUE
+export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64
+export PYTHON_MODULE=python_3.10.4
+export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp
+export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10
+export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib\ -lpython3.10\ -lpthread\ -ldl\ -lutil\ -lm\ -Xlinker\ -export-dynamic
+export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4
+# JY export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2
+export MET_HDF5=${HDF5_ROOT}
+export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.6.0/lib64
+# JY export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64
+# JY export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include
+export MET_GRIB2CLIB=${g2c_ROOT}/lib64
+export MET_GRIB2CINC=${G2C_INC}
+export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb
+export BUFRLIB_NAME=-lbufr_4
+export GRIB2CLIB_NAME=-lg2c
+# JY export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64
+export LIB_JASPER=${JASPER_LIBDIR}
+# JY export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib
+export LIB_LIBPNG=${LIBPNG_LIBDIR}
+# JY export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib
+export LIB_Z=${ZLIB_LIBDIR}
+export SET_D64BIT=FALSE

internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2

@@ -0,0 +1,27 @@
+help([[
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+local hierA        = hierarchyA(pkgNameVer,1)
+local compNameVer  = hierA[1]
+
+
+conflict(pkgName)
+
+local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules"
+
+local base = pathJoin(opt,compNameVer,pkgName,pkgVersion)
+
+prepend_path("PATH", pathJoin(base,"bin"))
+
+setenv("MET_ROOT", base)
+setenv("MET_BASE", pathJoin(base, "share", "met"))
+setenv("MET_VERSION", pkgVersion)
+
+whatis("Name: ".. pkgName)
+whatis("Version: " .. pkgVersion)
+whatis("Category: applications")
+whatis("Description: Model Evaluation Tools (MET)")

internal/scripts/installation/modulefiles/11.0.1_acorn

@@ -0,0 +1,34 @@
+#%Module######################################################################
+##
+##      Model Evaluation Tools
+##
+proc ModulesHelp { } {
+  puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.1
+     *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***"
+}
+
+# The intel compiler is required to run MET
+
+module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/
+module load intel python/3.10.4
+module load ve/evs/1.0
+module load netcdf/4.7.4
+module load hdf5/1.12.2
+module load bufr/11.5.0
+module load zlib/1.2.11
+module load jasper/2.0.25
+module load libpng/1.6.37
+module load gsl/2.7
+module load g2c/1.6.4
+
+set base /apps/sw_review/emc/MET/11.0.1
+set ver  11.0.1
+set share $base/share/met
+set lib_base $base
+
+prepend-path PATH $base/exec
+
+setenv METversion V$ver
+setenv MET_ROOT $base
+
+

internal/test_unit/xml/unit_python.xml

@@ -495,6 +495,25 @@
     </output>
   </test>
 
+  <!-- Invokes user-python logic to read a point obs -->
+  <test name="python_point2grid_pb2nc_TMP_user_python">
+    <exec>&MET_BIN;/point2grid</exec>
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
+    <param> \
+      'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \
+      G212 \
+      &OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc \
+      -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ &lt;0 ]; censor_val=[0];' \
+      -name TEMP \
+      -v 1
+    </param>
+    <output>
+      <grid_nc>&OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc</grid_nc>
+    </output>
+  </test>
+
   <test name="python_plot_point_obs_with_point_data">
     <exec>&MET_BIN;/plot_point_obs</exec>
     <env>

scripts/python/Makefile.am

@@ -27,6 +27,7 @@ pythonscriptsdir = $(pkgdatadir)/python
 
 pythonscripts_DATA = \
 	met_point_obs.py \
+	met_point_obs_nc.py \
 	read_ascii_numpy.py \
 	read_ascii_numpy_grid.py \
 	read_ascii_xarray.py \

scripts/python/Makefile.in

@@ -298,6 +298,7 @@ top_srcdir = @top_srcdir@
 pythonscriptsdir = $(pkgdatadir)/python
 pythonscripts_DATA = \
 	met_point_obs.py \
+	met_point_obs_nc.py \
 	read_ascii_numpy.py \
 	read_ascii_numpy_grid.py \
 	read_ascii_xarray.py \