diff --git a/.github/actions/run_tests/Dockerfile.run b/.github/actions/run_tests/Dockerfile.run index 7a2772f6f1..133783e8a6 100644 --- a/.github/actions/run_tests/Dockerfile.run +++ b/.github/actions/run_tests/Dockerfile.run @@ -7,5 +7,4 @@ ARG METPLUS_IMG_TAG=develop FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} COPY --from=env /usr/local/envs /usr/local/envs/ - COPY --from=env /usr/local/bin/conda /usr/local/bin/conda diff --git a/.github/actions/run_tests/Dockerfile.run_cartopy b/.github/actions/run_tests/Dockerfile.run_cartopy new file mode 100644 index 0000000000..58256ff2a6 --- /dev/null +++ b/.github/actions/run_tests/Dockerfile.run_cartopy @@ -0,0 +1,12 @@ +ARG METPLUS_ENV_TAG=cycloneplotter +ARG METPLUS_IMG_TAG=develop + +FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env + +ARG METPLUS_IMG_TAG=develop +FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG} + +COPY --from=env /usr/local/envs /usr/local/envs/ +COPY --from=env /usr/local/bin/conda /usr/local/bin/conda + +COPY --from=env /root/.local/share/cartopy /root/.local/share/cartopy/ \ No newline at end of file diff --git a/.github/jobs/get_metviewer.sh b/.github/jobs/get_metviewer.sh index 2dc87018e9..b20cdde333 100755 --- a/.github/jobs/get_metviewer.sh +++ b/.github/jobs/get_metviewer.sh @@ -4,7 +4,7 @@ export METVIEWER_DATA=$RUNNER_WORKSPACE export MYSQL_DIR=$RUNNER_WORKSPACE/mysql export METVIEWER_DIR=$RUNNER_WORKSPACE/output/metviewer -export METVIEWER_DOCKER_IMAGE=dtcenter/metviewer +export METVIEWER_DOCKER_IMAGE=dtcenter/metviewer:develop # install docker-compose apk add docker-compose diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index 1a54b84702..ff1fa35e9c 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -76,6 +76,8 @@ def main(): dockerfile_name = f'{dockerfile_name}_gempak' elif 'gfdl' in str(requirements).lower(): dockerfile_name = f'{dockerfile_name}_gfdl' + elif 'cartopy' in str(requirements).lower(): + dockerfile_name = f'{dockerfile_name}_cartopy' docker_build_cmd = ( f"docker build -t {run_tag} " diff --git a/ci/docker/docker_env/README.md b/ci/docker/docker_env/README.md index 93e37e7a34..dc40f5ad0c 100644 --- a/ci/docker/docker_env/README.md +++ b/ci/docker/docker_env/README.md @@ -163,6 +163,61 @@ will fail through the script, but it should be installed using pip3 afterwards. ``` ./scripts/metplotpy_env.sh metplus_base /home/met_test/.conda/envs/metplotpy/bin/pip3 install kaleido==0.2.1 +/home/met_test/.conda/envs/metplotpy/bin/python3 cartopy_feature_download.py cultural physical +rm cartopy_feature_download.py +``` + +#### To install METplotpy and METcalcpy packages in environment + +``` +cd /home/met_test + +# git clone not necessary if repo is already available +git clone https://github.com/dtcenter/METplotpy +git clone https://github.com/dtcenter/METcalcpy + +cd /home/met_test/METplotpy +git checkout develop +/home/met_test/.conda/envs/metplotpy/bin/pip3 install . + +cd /home/met_test/METcalcpy +git checkout develop +/home/met_test/.conda/envs/metplotpy/bin/pip3 install . +``` + +## weatherregime (from metplotpy_env) + +### Docker + +``` +docker build -t dtcenter/metplus-envs:weatherregime --build-arg BASE_ENV=metplotpy --build-arg ENV_NAME=weatherregime . +docker push dtcenter/metplus-envs:weatherregime +``` + +### Local + +``` +./scripts/weatherregime_env.sh metplotpy +/home/met_test/.conda/envs/weatherregime/bin/python3 cartopy_feature_download.py cultural physical +rm cartopy_feature_download.py +``` + +#### To install METplotpy and METcalcpy packages in environment + +``` +cd /home/met_test + +# git clone not necessary if repo is already available +git clone https://github.com/dtcenter/METplotpy +git clone https://github.com/dtcenter/METcalcpy + +cd /home/met_test/METplotpy +git checkout develop +/home/met_test/.conda/envs/weatherregime/bin/pip3 install . + +cd /home/met_test/METcalcpy +git checkout develop +/home/met_test/.conda/envs/weatherregime/bin/pip3 install . ``` ## cycloneplotter (from metplus_base) diff --git a/ci/docker/docker_env/scripts/cycloneplotter_env.sh b/ci/docker/docker_env/scripts/cycloneplotter_env.sh index 477f428a5c..17706c39d0 100755 --- a/ci/docker/docker_env/scripts/cycloneplotter_env.sh +++ b/ci/docker/docker_env/scripts/cycloneplotter_env.sh @@ -23,3 +23,7 @@ conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} conda install -y --name ${ENV_NAME} -c conda-forge cartopy==0.17.0 conda install -y --name ${ENV_NAME} -c conda-forge matplotlib==3.3.0 conda install -y --name ${ENV_NAME} -c conda-forge pandas + +yum -y install wget +wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py +/usr/local/envs/${ENV_NAME}/bin/python3 cartopy_feature_download.py cultural physical diff --git a/ci/docker/docker_env/scripts/metplotpy_env.sh b/ci/docker/docker_env/scripts/metplotpy_env.sh index 3bead781c1..1db44769ee 100755 --- a/ci/docker/docker_env/scripts/metplotpy_env.sh +++ b/ci/docker/docker_env/scripts/metplotpy_env.sh @@ -42,3 +42,7 @@ conda install -y --name ${ENV_NAME} -c conda-forge xarray==0.17 conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.6 conda install -y --name ${ENV_NAME} -c conda-forge pyyaml /usr/local/envs/${ENV_NAME}/bin/pip3 install kaleido==0.2.1 + +yum -y install wget +wget https://raw.githubusercontent.com/SciTools/cartopy/master/tools/cartopy_feature_download.py +/usr/local/envs/${ENV_NAME}/bin/python3 cartopy_feature_download.py cultural physical diff --git a/ci/docker/docker_env/scripts/weatherregime_env.sh b/ci/docker/docker_env/scripts/weatherregime_env.sh index bb2f7c81d3..16c34b456e 100755 --- a/ci/docker/docker_env/scripts/weatherregime_env.sh +++ b/ci/docker/docker_env/scripts/weatherregime_env.sh @@ -2,23 +2,13 @@ ################################################################################ # Environment: weatherregime -# Last Updated: 2021-06-29 (mccabe@ucar.edu) +# Last Updated: 2021-09-16 (mccabe@ucar.edu) # Notes: Adds Python packages needed to run weather regime use case # METplotpy and METcalcpy -# Uses pip to install kaleido because -# could not install via Conda (glibc conflict) # Python Packages: -# matplotlib==3.3.0 -# scipy==1.5.1 -# plotly==4.9.0 -# pingouin==0.3.8 -# cartopy==0.18.0 -# eofs==1.3.0 -# cmocean==2.0 -# xarray==0.17 -# netcdf4==1.5.6 -# pyyaml==? -# python-kaleido==0.2.1 +# All packages from metplotpy_env +# scikit-learn==0.24.2 +# eofs==1.4.0 # # Other Content: None ################################################################################ @@ -32,7 +22,9 @@ BASE_ENV=$1 conda create -y --clone ${BASE_ENV} --name ${ENV_NAME} conda install -y --name ${ENV_NAME} -c conda-forge scikit-learn==0.24.2 -#conda install -y --name ${ENV_NAME} -c conda-forge scipy==1.5.4 conda install -y --name ${ENV_NAME} -c conda-forge eofs==1.4.0 -conda install -y --name ${ENV_NAME} -c conda-forge netcdf4==1.5.7 -#conda install -y --name ${ENV_NAME} -c conda-forge numpy==1.19.5 + +rm cartopy_feature_download.py +yum -y install wget +wget https://raw.githubusercontent.com/georgemccabe/cartopy/master/tools/cartopy_feature_download.py +/usr/local/envs/${ENV_NAME}/bin/python3 cartopy_feature_download.py cultural physical cultural-extra diff --git a/docs/Users_Guide/systemconfiguration.rst b/docs/Users_Guide/systemconfiguration.rst index cf5df4a0e8..fa8db50ddb 100644 --- a/docs/Users_Guide/systemconfiguration.rst +++ b/docs/Users_Guide/systemconfiguration.rst @@ -2282,6 +2282,17 @@ time_summary.step and time_summary.width | | | :term:`PB2NC_TIME_SUMMARY_WIDTH` = 3600 | +------------------+-------------------------------------------+ +pb_report_type +^^^^^^^^^^^^^^ + ++------------------+----------------------------------------------------------------------------------------------------------------------------------------+ +| Old (Incorrect): | pb_report_type = [ 120, 220, 221, 122, 222, 223, 224, 133, 233, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 ]; | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------+ +| New (Correct): | pb_report_type = []; | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------+ +| METplus Config: | :term:`PB2NC_PB_REPORT_TYPE` = 120, 220, 221, 122, 222, 223, 224, 133, 233, 188, 288, 180, 280, 181, 182, 281, 282, 183, 284, 187, 287 | ++------------------+----------------------------------------------------------------------------------------------------------------------------------------+ + PointStatConfig --------------- diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal_tests/use_cases/all_use_cases.txt index c8e3aa8fcc..bf65342aa6 100644 --- a/internal_tests/use_cases/all_use_cases.txt +++ b/internal_tests/use_cases/all_use_cases.txt @@ -1,5 +1,5 @@ Category: met_tool_wrapper -0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3:: cycloneplotter_env +0::CyclonePlotter::met_tool_wrapper/CyclonePlotter/CyclonePlotter.conf,user_env_vars.MET_PYTHON_EXE=python3:: cycloneplotter_env,cartopy 1::PCPCombine_python_embedding:: met_tool_wrapper/PCPCombine/PCPCombine_python_embedding.conf:: h5py_env,py_embed 2::ASCII2NC:: met_tool_wrapper/ASCII2NC/ASCII2NC.conf 3::met_tool_wrapper/ASCII2NC/ASCII2NC_python_embedding.conf @@ -117,16 +117,16 @@ Category: precipitation Category: s2s 0::GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast:: model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf:: netcdf4_env -1::UserScript_fcstGFS_obsERA_Blocking:: model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking.conf:: metplotpy_env,metplus -2::UserScript_obsERA_obsOnly_Blocking:: model_applications/s2s/UserScript_obsERA_obsOnly_Blocking.conf:: metplotpy_env,metplus -3::UserScript_obsERA_obsOnly_WeatherRegime:: model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf:: weatherregime_env,metplus -4::TCGen_fcstGFSO_obsBDECKS_GDF_TDF:: model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf:: metplotpy_env,metplus -5::UserScript_obsPrecip_obsOnly_Hovmoeller:: model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf:: metplotpy_env +1::UserScript_fcstGFS_obsERA_Blocking:: model_applications/s2s/UserScript_fcstGFS_obsERA_Blocking.conf:: metplotpy_env,cartopy,metplus +2::UserScript_obsERA_obsOnly_Blocking:: model_applications/s2s/UserScript_obsERA_obsOnly_Blocking.conf:: metplotpy_env,cartopy,metplus +3::UserScript_obsERA_obsOnly_WeatherRegime:: model_applications/s2s/UserScript_obsERA_obsOnly_WeatherRegime.conf:: weatherregime_env,cartopy,metplus +4::TCGen_fcstGFSO_obsBDECKS_GDF_TDF:: model_applications/s2s/TCGen_fcstGFSO_obsBDECKS_GDF_TDF.conf:: metplotpy_env,cartopy,metplus +5::UserScript_obsPrecip_obsOnly_Hovmoeller:: model_applications/s2s/UserScript_obsPrecip_obsOnly_Hovmoeller.conf:: metplotpy_env,cartopy 6:: UserScript_obsPrecip_obsOnly_CrossSpectraPlot:: model_applications/s2s/UserScript_obsPrecip_obsOnly_CrossSpectraPlot.conf:: spacetime_env 7:: UserScript_fcstGFS_obsERA_PhaseDiagram:: model_applications/s2s/UserScript_fcstGFS_obsERA_PhaseDiagram.conf:: spacetime_env 8:: UserScript_fcstGFS_obsERA_OMI:: model_applications/s2s/UserScript_fcstGFS_obsERA_OMI.conf:: spacetime_env, metdatadb 9:: UserScript_fcstGFS_obsERA_RMM:: model_applications/s2s/UserScript_fcstGFS_obsERA_RMM.conf:: spacetime_env, metdatadb -10::UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,metplus +10::UserScript_fcstGFS_obsERA_WeatherRegime:: model_applications/s2s/UserScript_fcstGFS_obsERA_WeatherRegime.conf:: weatherregime_env,cartopy,metplus Category: space_weather 0::GridStat_fcstGloTEC_obsGloTEC_vx7:: model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf @@ -135,6 +135,6 @@ Category: space_weather Category: tc_and_extra_tc 0::TCRMW_fcstGFS_fcstOnly_gonzalo:: model_applications/tc_and_extra_tc/TCRMW_fcstGFS_fcstOnly_gonzalo.conf -1::CyclonePlotter_fcstGFS_obsGFS_OPC:: model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf:: cycloneplotter_env +1::CyclonePlotter_fcstGFS_obsGFS_OPC:: model_applications/tc_and_extra_tc/CyclonePlotter_fcstGFS_obsGFS_OPC.conf:: cycloneplotter_env,cartopy 2::UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF:: model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF.conf,model_applications/tc_and_extra_tc/UserScript_ASCII2NC_PointStat_fcstHAFS_obsFRD_NetCDF/ci_overrides.conf 3::GridStat_fcstHAFS_obsTDR_NetCDF:: model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf:: py_embed diff --git a/metplus/util/config_metplus.py b/metplus/util/config_metplus.py index 30c24ee34d..3a6df221d3 100644 --- a/metplus/util/config_metplus.py +++ b/metplus/util/config_metplus.py @@ -56,16 +56,16 @@ '''!@var METPLUS_BASE The METplus installation directory ''' -METPLUS_BASE = str(Path(__file__).parents[2]) +METPLUS_BASE = os.path.realpath(str(Path(__file__).parents[2])) -'''!@var PARM_BASE -The parameter directory - set to METPLUS_BASE/parm unless the -METPLUS_PARM_BASE environment variable is set -''' +# name of directory under METPLUS_BASE that contains config files +PARM_DIR = 'parm' + +# set parm base to METPLUS_BASE/parm unless METPLUS_PARM_BASE env var is set PARM_BASE = os.environ.get('METPLUS_PARM_BASE', - os.path.join(METPLUS_BASE, 'parm')) + os.path.join(METPLUS_BASE, PARM_DIR)) -# name of directory under PARM_BASE that contains defaults +# name of directory under PARM_DIR that contains defaults METPLUS_CONFIG_DIR = 'metplus_config' # default METplus configuration files that are sourced first @@ -91,9 +91,8 @@ def get_default_config_list(parm_base=None): @returns list of full paths to default METplus config files """ default_config_list = [] - # set parm to real path of PARM_BASE if not provided as argument if parm_base is None: - parm_base = os.path.realpath(PARM_BASE) + parm_base = PARM_BASE conf_dir = os.path.join(parm_base, METPLUS_CONFIG_DIR) @@ -107,14 +106,13 @@ def get_default_config_list(parm_base=None): if not default_config_list: print(f"ERROR: No default config files found in {conf_dir}") - print("Check if METPLUS_PARM_BASE is set and unset it if so") sys.exit(1) return default_config_list -def setup(config_inputs, logger=None, base_confs=None): +def setup(args, logger=None, base_confs=None): """!The METplus setup function. - @param config_inputs list of configuration files or configuration + @param args list of configuration files or configuration variable overrides. Reads all configuration inputs and returns a configuration object. """ @@ -128,124 +126,90 @@ def setup(config_inputs, logger=None, base_confs=None): logger.info('Starting METplus configuration setup.') - if isinstance(config_inputs, str): - config_list = [config_inputs] - else: - config_list = config_inputs - - # parm, is path to parm directory - # infiles, list of input conf files to be read and processed - # moreopt, dictionary of conf file settings, passed in from command line. - (parm, infiles, moreopt) = \ - parse_launch_args(config_list, - logger, - base_confs=base_confs) - - config = launch(infiles, moreopt) + override_list = parse_launch_args(args, logger) - # save list of user configuration files in a variable - config.set('config', 'METPLUS_CONFIG_FILES', ','.join(config_list)) + # add default config files to override list + override_list = base_confs + override_list + config = launch(override_list) - logger.info('Completed METplus configuration setup.') + logger.debug('Completed METplus configuration setup.') return config -# def parse_launch_args(args,usage,logger,PARM_BASE=None): -# This is intended to be use to gather all the conf files on the -# command line, along with overide options on the command line. -# This includes the default conf files metplus.conf, metplus.override.conf -# along with, -c some.conf and any other conf files... -# These are than used by def launch to create a single metplus final conf file -# that would be used by all tasks. -def parse_launch_args(args, logger, base_confs=None): - """!Parsed arguments to scripts that launch the METplus system. - - This is the argument parser for the config_metplus.py - script. It parses the arguments (in args). - If something goes wrong, this function calls - sys.exit(1) or sys.exit(2). +def parse_launch_args(args, logger): + """! Parsed arguments to scripts that launch the METplus wrappers. Options: - * section.variable=value --- set this value in this section, no matter what - * /path/to/file.conf --- read this conf file after the default conf files. - - Later conf files override earlier ones. The conf files read in - are: - * metplus.conf + * section.variable=value --- set this value in this section + * /path/to/file.conf --- read this conf file after the default conf files @param args the script arguments, after script-specific ones are removed - @param logger a logging.Logger for log messages""" - - parm = os.path.realpath(PARM_BASE) - if base_confs is None: - base_confs = get_default_config_list() - - # Files in this list, that don't exist or are empty, - # will be silently ignored. - # infiles=[ os.path.join(parm, 'metplus.conf'), - # os.path.join(parm, 'metplus.override.conf') - # ] - infiles = list() - for base_conf in base_confs: - infiles.append(base_conf) + @param logger a logging.Logger for log messages + @returns tuple containing path to parm directory, list of config files and + collections.defaultdict of explicit config overrides + """ + if not args: + return [] - moreopt = collections.defaultdict(dict) + if isinstance(args, str): + args = [args] - if args is None: - return (parm, infiles, moreopt) + override_list = [] # Now look for any option and conf file arguments: bad = False - for iarg in range(len(args)): + for arg in args: m = re.match(r'''(?x) (?P
[a-zA-Z][a-zA-Z0-9_]*) \.(?P