Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bugfix 2428 python embedding with MET_PYTHON_EXE defined #2462

Merged
merged 14 commits into from
Feb 24, 2023
Merged
2 changes: 2 additions & 0 deletions data/wrappers/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,10 @@ wrappers_DATA = \
set_python_env.py \
read_tmp_dataplane.py \
read_tmp_ascii.py \
read_tmp_point_nc.py \
write_tmp_dataplane.py \
write_tmp_point.py \
write_tmp_point_nc.py \
write_tmp_mpr.py

EXTRA_DIST = ${wrappers_DATA}
Expand Down
2 changes: 2 additions & 0 deletions data/wrappers/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,10 @@ wrappers_DATA = \
set_python_env.py \
read_tmp_dataplane.py \
read_tmp_ascii.py \
read_tmp_point_nc.py \
write_tmp_dataplane.py \
write_tmp_point.py \
write_tmp_point_nc.py \
write_tmp_mpr.py

EXTRA_DIST = ${wrappers_DATA}
Expand Down
30 changes: 30 additions & 0 deletions data/wrappers/read_tmp_point_nc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
########################################################################
#
# Reads temporary file into memory.
#
# usage: /path/to/python read_tmp_dataplane.py dataplane.tmp
#
########################################################################

import os
import sys

met_base_dir = os.getenv('MET_BASE',None)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think these 3 lines are necessary since the path is appended using the relative path of the script below.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MET_BASE was removed at read_tmp_point_nc.py and write_tmp_point_nc.py

if met_base_dir is not None:
sys.path.append(os.path.join(met_base_dir, 'python'))

# add share/met/python directory to system path to find met_point_obs
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir, 'python')))
from met_point_obs import met_point_obs
from met_point_obs_nc import nc_point_obs

netcdf_filename = sys.argv[1]

# read NetCDF file
print('{p} reading{f}'.format(p=met_point_obs.get_prompt(), f=netcdf_filename))
point_obs_data = nc_point_obs()
point_obs_data.read_data(netcdf_filename)

met_point_data = point_obs_data.get_point_data()
met_point_data['met_point_data'] = point_obs_data
59 changes: 59 additions & 0 deletions data/wrappers/write_tmp_point_nc.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
########################################################################
#
# Adapted from a script provided by George McCabe
# Adapted by Randy Bullock
#
# usage: /path/to/python write_tmp_point.py \
# tmp_output_filename <user_python_script>.py <args>
#
########################################################################

import os
import sys
import importlib.util

met_base_dir = os.getenv('MET_BASE',None)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think these 3 lines are necessary since the path is appended using the relative path of the script below.

if met_base_dir is not None:
sys.path.append(os.path.join(met_base_dir, 'python'))

# add share/met/python directory to system path to find met_point_obs
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__),
os.pardir, 'python')))

from met_point_obs import met_point_obs
from met_point_obs_nc import nc_point_obs

PROMPT = met_point_obs.get_prompt()
print("{p} Python Script:\t".format(p=PROMPT) + repr(sys.argv[0]))
print("{p} User Command:\t".format(p=PROMPT) + repr(' '.join(sys.argv[2:])))
print("{p} Temporary File:\t".format(p=PROMPT) + repr(sys.argv[1]))

tmp_filename = sys.argv[1]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A lot of this logic looks to be the same as other scripts in the data/wrappers directory. We should consider pulling out duplicate code into a function. This isn't required for this PR and could be done during a bigger refactor of the Python Embedding logic.

pyembed_module_name = sys.argv[2]
sys.argv = sys.argv[2:]

# append user script dir to system path
pyembed_dir, pyembed_file = os.path.split(pyembed_module_name)
if pyembed_dir:
sys.path.insert(0, pyembed_dir)

if not pyembed_module_name.endswith('.py'):
pyembed_module_name += '.py'

user_base = os.path.basename(pyembed_module_name).replace('.py','')

spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name)
met_in = importlib.util.module_from_spec(spec)
spec.loader.exec_module(met_in)

if hasattr(met_in, 'point_obs_data'):
met_in.point_obs_data.save_ncfile(tmp_filename)
else:
if hasattr(met_in.met_point_data, 'point_obs_data'):
met_in.met_point_data['point_obs_data'].save_ncfile(tmp_filename)
else:
tmp_point_obs = nc_point_obs()
tmp_point_obs.put_data(met_in.met_point_data)
tmp_point_obs.save_ncfile(tmp_filename)

#print('{p} writing {f}'.format(p=PROMPT, f=tmp_filename))
19 changes: 19 additions & 0 deletions internal/test_unit/xml/unit_python.xml
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,25 @@
</output>
</test>

<!-- Invokes user-python logic to read a point obs -->
<test name="python_point2grid_pb2nc_TMP_user_python">
<exec>&MET_BIN;/point2grid</exec>
<env>
<pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
</env>
<param> \
'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \
G212 \
&OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc \
-field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ &lt;0 ]; censor_val=[0];' \
-name TEMP \
-v 1
</param>
<output>
<grid_nc>&OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc</grid_nc>
</output>
</test>

<test name="python_plot_point_obs_with_point_data">
<exec>&MET_BIN;/plot_point_obs</exec>
<env>
Expand Down
1 change: 1 addition & 0 deletions scripts/python/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ pythonscriptsdir = $(pkgdatadir)/python

pythonscripts_DATA = \
met_point_obs.py \
met_point_obs_nc.py \
read_ascii_numpy.py \
read_ascii_numpy_grid.py \
read_ascii_xarray.py \
Expand Down
1 change: 1 addition & 0 deletions scripts/python/Makefile.in
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,7 @@ top_srcdir = @top_srcdir@
pythonscriptsdir = $(pkgdatadir)/python
pythonscripts_DATA = \
met_point_obs.py \
met_point_obs_nc.py \
read_ascii_numpy.py \
read_ascii_numpy_grid.py \
read_ascii_xarray.py \
Expand Down
Loading