-
Notifications
You must be signed in to change notification settings - Fork 224
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
**BREAKING** pygmt.grdcut: Refactor to store output in virtualfiles for grids #3115
base: main
Are you sure you want to change the base?
Changes from 83 commits
bcf43f0
a052a1a
59d523c
56a6d65
3315324
cea3374
80d9837
22fba56
f71e79c
4cce4a2
e02b650
f3d4b1f
4390136
5f25669
a3c6c14
5888e10
798e658
3dbf2f2
3a24ebd
4eee7e6
5e390d4
003383d
606ac7e
c6cdcc8
377941a
20617f5
a998718
86cab44
6031bab
eb0af2d
7f6ca7d
21b194a
e7eaf5c
e3c8569
1c8312c
3913430
812a225
90bd29e
ab77187
6f3e474
5b07dd9
31272ab
6b860bf
5a09329
279595b
7def4b5
be175d8
0bf9368
7d437be
268e34e
86765e1
86f3ffa
cc28247
1e2c973
734dc28
919dc00
b1eacf1
aa4fdc9
c87a3ec
a20d8a2
926427b
c73328e
2825eae
bf9275c
fb97daa
15b8d53
8433e78
3e3a6f3
12ef40a
f64fbb8
e9cb0a5
4f2ae48
d49afed
a97d0b3
f70bec0
2fd13fb
9972ba1
ac6b7c3
7c32d41
9ec00be
f3a2f8e
3c12e2b
f852b0d
5f7683c
bb1a0b0
584b5af
b19ba00
cceb929
fac51f1
517ee52
d5ec308
7ca58fb
66580d4
194ee90
473428c
84b2ed4
7968096
90d5210
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -1942,6 +1942,7 @@ def virtualfile_out( | |
"grid": ("GMT_IS_GRID", "GMT_IS_SURFACE"), | ||
"image": ("GMT_IS_IMAGE", "GMT_IS_SURFACE"), | ||
}[kind] | ||
# For unknown reasons, 'GMT_OUT' crashes for 'image' kind. | ||
direction = "GMT_OUT|GMT_IS_REFERENCE" if kind == "image" else "GMT_OUT" | ||
with self.open_virtualfile(family, geometry, direction, None) as vfile: | ||
yield vfile | ||
|
@@ -2311,3 +2312,35 @@ def extract_region(self) -> np.ndarray: | |
if status != 0: | ||
raise GMTCLibError("Failed to extract region from current figure.") | ||
return region | ||
|
||
|
||
def _raster_kind(raster: str) -> Literal["grid", "image"]: | ||
""" | ||
Determine the raster kind. | ||
|
||
Examples | ||
-------- | ||
>>> _raster_kind("@earth_relief_01d") | ||
'grid' | ||
>>> _raster_kind("@static_earth_relief.nc") | ||
'grid' | ||
>>> _raster_kind("@earth_day_01d") | ||
'image' | ||
>>> _raster_kind("@hotspots.txt") | ||
'grid' | ||
""" | ||
# The logic here is because: an image can be read into a grid container, but a grid | ||
# can't be read into an image container. So, try to read the file as an image first. | ||
# If fails, try to read it as a grid. | ||
with Session() as lib: | ||
try: | ||
img = lib.read_data(infile=raster, kind="image", mode="GMT_CONTAINER_ONLY") | ||
return "image" if img.contents.header.contents.n_bands == 3 else "grid" | ||
except GMTCLibError: | ||
pass | ||
try: | ||
_ = lib.read_data(infile=raster, kind="grid", mode="GMT_CONTAINER_ONLY") | ||
return "grid" | ||
except GMTCLibError: | ||
pass | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just double-checking that there's not a function in GMT C already to determine grid/image type? I know there's There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Oo, magic byte parsing 😆 If that function becomes public someday, maybe we'll use it. |
||
return "grid" # Fallback to "grid" and let GMT determine the type. |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -4,21 +4,21 @@ | |
|
||
import xarray as xr | ||
from pygmt.clib import Session | ||
from pygmt.clib.session import _raster_kind | ||
from pygmt.exceptions import GMTInvalidInput | ||
from pygmt.helpers import ( | ||
GMTTempFile, | ||
build_arg_list, | ||
data_kind, | ||
fmt_docstring, | ||
kwargs_to_strings, | ||
use_alias, | ||
) | ||
from pygmt.io import load_dataarray | ||
|
||
__doctest_skip__ = ["grdcut"] | ||
|
||
|
||
@fmt_docstring | ||
@use_alias( | ||
G="outgrid", | ||
R="region", | ||
J="projection", | ||
N="extend", | ||
|
@@ -28,9 +28,9 @@ | |
f="coltypes", | ||
) | ||
@kwargs_to_strings(R="sequence") | ||
def grdcut(grid, **kwargs) -> xr.DataArray | None: | ||
def grdcut(grid, outgrid: str | None = None, **kwargs) -> xr.DataArray | None: | ||
r""" | ||
Extract subregion from a grid. | ||
Extract subregion from a grid or image. | ||
|
||
Produce a new ``outgrid`` file which is a subregion of ``grid``. The | ||
subregion is specified with ``region``; the specified range must not exceed | ||
|
@@ -100,13 +100,24 @@ | |
>>> # 12° E to 15° E and a latitude range of 21° N to 24° N | ||
>>> new_grid = pygmt.grdcut(grid=grid, region=[12, 15, 21, 24]) | ||
""" | ||
with GMTTempFile(suffix=".nc") as tmpfile: | ||
with Session() as lib: | ||
with lib.virtualfile_in(check_kind="raster", data=grid) as vingrd: | ||
if (outgrid := kwargs.get("G")) is None: | ||
kwargs["G"] = outgrid = tmpfile.name # output to tmpfile | ||
lib.call_module( | ||
module="grdcut", args=build_arg_list(kwargs, infile=vingrd) | ||
) | ||
# Determine the output data kind based on the input data kind. | ||
inkind = data_kind(grid) | ||
match inkind: | ||
case "image" | "grid": | ||
outkind = inkind | ||
case "file": | ||
outkind = _raster_kind(grid) | ||
case "_": | ||
msg = f"Unsupported data type {type(grid)}." | ||
raise GMTInvalidInput(msg) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lines 117-118 should be caught by the |
||
|
||
return load_dataarray(outgrid) if outgrid == tmpfile.name else None | ||
with Session() as lib: | ||
with ( | ||
lib.virtualfile_in(check_kind="raster", data=grid) as vingrd, | ||
lib.virtualfile_out(kind=outkind, fname=outgrid) as voutgrd, | ||
): | ||
kwargs["G"] = voutgrd | ||
lib.call_module(module="grdcut", args=build_arg_list(kwargs, infile=vingrd)) | ||
return lib.virtualfile_to_raster( | ||
vfname=voutgrd, kind=outkind, outgrid=outgrid | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,85 @@ | ||
""" | ||
Test pygmt.grdcut on images. | ||
""" | ||
|
||
from pathlib import Path | ||
|
||
import numpy as np | ||
import pytest | ||
import xarray as xr | ||
from pygmt import grdcut, which | ||
from pygmt.helpers import GMTTempFile | ||
|
||
try: | ||
import rioxarray | ||
|
||
_HAS_RIOXARRAY = True | ||
except ImportError: | ||
_HAS_RIOXARRAY = False | ||
|
||
|
||
@pytest.fixture(scope="module", name="region") | ||
def fixture_region(): | ||
""" | ||
Set the data region. | ||
""" | ||
return [-53, -49, -20, -17] | ||
|
||
|
||
@pytest.fixture(scope="module", name="expected_image") | ||
def fixture_expected_image(): | ||
""" | ||
Load the expected grdcut image result. | ||
""" | ||
return xr.DataArray( | ||
data=np.array( | ||
[ | ||
[[90, 93, 95, 90], [91, 90, 91, 91], [91, 90, 89, 90]], | ||
[[87, 88, 88, 89], [88, 87, 86, 85], [90, 90, 89, 88]], | ||
[[48, 49, 49, 45], [49, 48, 47, 45], [48, 47, 48, 46]], | ||
], | ||
dtype=np.uint8, | ||
), | ||
coords={ | ||
"band": [1, 2, 3], | ||
seisman marked this conversation as resolved.
Show resolved
Hide resolved
|
||
"x": [-52.5, -51.5, -50.5, -49.5], | ||
"y": [-17.5, -18.5, -19.5], | ||
}, | ||
dims=["band", "y", "x"], | ||
attrs={ | ||
"scale_factor": 1.0, | ||
"add_offset": 0.0, | ||
}, | ||
) | ||
|
||
|
||
@pytest.mark.benchmark | ||
def test_grdcut_image_file(region, expected_image): | ||
""" | ||
Test grdcut on an input image file. | ||
""" | ||
result = grdcut("@earth_day_01d_p", region=region) | ||
xr.testing.assert_allclose(a=result, b=expected_image) | ||
|
||
|
||
@pytest.mark.skipif(not _HAS_RIOXARRAY, reason="rioxarray is not installed") | ||
seisman marked this conversation as resolved.
Show resolved
Hide resolved
|
||
def test_grdcut_image_dataarray(region, expected_image): | ||
""" | ||
Test grdcut on an input xarray.DataArray object. | ||
""" | ||
raster = rioxarray.open_rasterio(which("@earth_day_01d", download="a")).load() | ||
seisman marked this conversation as resolved.
Show resolved
Hide resolved
|
||
result = grdcut(raster, region=region) | ||
xr.testing.assert_allclose(a=result, b=expected_image) | ||
|
||
|
||
def test_grdcut_image_file_in_file_out(region, expected_image): | ||
""" | ||
Test grdcut on an input image file and outputs to another image file. | ||
""" | ||
with GMTTempFile(suffix=".tif") as tmp: | ||
result = grdcut("@earth_day_01d_p", region=region, outgrid=tmp.name) | ||
assert result is None | ||
assert Path(tmp.name).stat().st_size > 0 | ||
if _HAS_RIOXARRAY: | ||
raster = rioxarray.open_rasterio(which("@earth_day_01d", download="a")).load() | ||
xr.testing.assert_allclose(a=raster, b=expected_image) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It turns out the logic here is not 100% correct.
libgdal-netcdf
orlibgdal-hdf5
) is installed. xref: Add libgdal-hdf5 dependency conda-forge/gmt-feedstock#293There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The first try-except check is to try reading into a
GMT_IMAGE
struct, and return "image" if there are 3 bands, and "grid" if there is only 1 band. Even if grids can be read intoGMT_IMAGE
, we will still call it a grid if it has 1-band right?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Trying to refresh my memory about what's happening:
GMT_GRID
: Works!GMT_GRID
: Only the first band is read.GMT_IMAGE
: It depends onlibgdal-hdf5
being installed. Read as an single-band image if installed, otherwise raise errors.GMT_IMAGE
: works.If
libgdal-hdf5
is not installed,_raster_kind
first read it as an image, which fails and reports following errors:then it tries to read it as a grid, which succeeds. But we still see the error messages above. I believe we need to find a way to suppress these errors or find an alternative way to determine the data kind.