Skip to content

Commit

Permalink
NAME loader fixes (#4411)
Browse files Browse the repository at this point in the history
* Convert size 1 array dtype without becoming scalar

* Ensure z coords are recognised by guess_coord_axis

* Test time coord creation

* Allow integers in lat-lon headings

* Update test results

* What's new
  • Loading branch information
vsherratt authored Nov 12, 2021
1 parent 2dda3a9 commit ab1e699
Show file tree
Hide file tree
Showing 9 changed files with 257 additions and 43 deletions.
5 changes: 4 additions & 1 deletion docs/src/whatsnew/latest.rst
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ This document explains the changes made to Iris for this release
🐛 Bugs Fixed
=============


#. `@rcomer`_ fixed :meth:`~iris.cube.Cube.intersection` for special cases where
one cell's bounds align with the requested maximum and negative minimum, fixing
:issue:`4221`. (:pull:`4278`)
Expand All @@ -114,6 +113,10 @@ This document explains the changes made to Iris for this release
to indicate that the value of a scalar coordinate may be mismatched, rather than the metadata
(:issue:`4096`, :pull:`4387`)

#. `@bsherratt`_ fixed a regression to the NAME file loader introduced in 3.0.4,
as well as some long-standing bugs with vertical coordinates and number
formats. (:pull:`4411`)


💣 Incompatible Changes
=======================
Expand Down
55 changes: 33 additions & 22 deletions lib/iris/fileformats/name_loaders.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,27 +160,37 @@ def _build_lat_lon_for_NAME_timeseries(column_headings):
the provided column_headings dictionary.
"""
pattern = re.compile(r"\-?[0-9]*\.[0-9]*")
new_Xlocation_column_header = []
for t in column_headings["X"]:
if "Lat-Long" in t:
matches = pattern.search(t)
new_Xlocation_column_header.append(float(matches.group(0)))
else:
new_Xlocation_column_header.append(t)
column_headings["X"] = new_Xlocation_column_header
# Pattern to match a number
pattern = re.compile(
r"""
[-+]? # Optional sign
(?:
\d+\.\d* # Float: integral part required
|
\d*\.\d+ # Float: fractional part required
|
\d+ # Integer
)
(?![0-9.]) # Not followed by a numeric character
""",
re.VERBOSE,
)

# Extract numbers from the X and Y column headings, which are currently
# strings of the form "X = -1.9 Lat-Long"
for key in ("X", "Y"):
new_headings = []
for heading in column_headings[key]:
match = pattern.search(heading)
if match and "Lat-Long" in heading:
new_headings.append(float(match.group(0)))
else:
new_headings.append(heading)
column_headings[key] = new_headings

lon = NAMECoord(
name="longitude", dimension=None, values=column_headings["X"]
)

new_Ylocation_column_header = []
for t in column_headings["Y"]:
if "Lat-Long" in t:
matches = pattern.search(t)
new_Ylocation_column_header.append(float(matches.group(0)))
else:
new_Ylocation_column_header.append(t)
column_headings["Y"] = new_Ylocation_column_header
lat = NAMECoord(
name="latitude", dimension=None, values=column_headings["Y"]
)
Expand Down Expand Up @@ -387,6 +397,7 @@ def _cf_height_from_name(z_coord, lower_bound=None, upper_bound=None):
standard_name=standard_name,
long_name=long_name,
bounds=bounds,
attributes={"positive": "up"},
)

return coord
Expand Down Expand Up @@ -478,7 +489,7 @@ def _generate_cubes(
coord_units = _parse_units("FL")
if coord.name == "time":
coord_units = time_unit
pts = np.float_(time_unit.date2num(coord.values))
pts = time_unit.date2num(coord.values).astype(float)

if coord.dimension is not None:
if coord.name == "longitude":
Expand All @@ -505,7 +516,7 @@ def _generate_cubes(
):
dt = coord.values - field_headings["Av or Int period"]
bnds = time_unit.date2num(np.vstack((dt, coord.values)).T)
icoord.bounds = np.float_(bnds)
icoord.bounds = bnds.astype(float)
else:
icoord.guess_bounds()
cube.add_dim_coord(icoord, coord.dimension)
Expand All @@ -522,7 +533,7 @@ def _generate_cubes(
):
dt = coord.values - field_headings["Av or Int period"]
bnds = time_unit.date2num(np.vstack((dt, coord.values)).T)
icoord.bounds = np.float_(bnds[i, :])
icoord.bounds = bnds[i, :].astype(float)
cube.add_aux_coord(icoord)

# Headings/column headings which are encoded elsewhere.
Expand Down Expand Up @@ -1250,7 +1261,7 @@ def load_NAMEIII_trajectory(filename):

long_name = units = None
if isinstance(values[0], datetime.datetime):
values = np.float_(time_unit.date2num(values))
values = time_unit.date2num(values).astype(float)
units = time_unit
if name == "Time":
name = "time"
Expand Down
30 changes: 25 additions & 5 deletions lib/iris/tests/results/name/NAMEIII_field.cml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,11 @@
<dimCoord bounds="[[349215.0, 349219.0]]" id="cb784457" points="[349219.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -112,7 +116,11 @@
<dimCoord bounds="[[349218.0, 349219.0]]" id="cb784457" points="[349219.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -172,7 +180,11 @@
<dimCoord bounds="[[349215.0, 349219.0]]" id="cb784457" points="[349219.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -232,7 +244,11 @@
<dimCoord bounds="[[349215.0, 349219.0]]" id="cb784457" points="[349219.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -292,7 +308,11 @@
<dimCoord bounds="[[349215.0, 349219.0]]" id="cb784457" points="[349219.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down
30 changes: 25 additions & 5 deletions lib/iris/tests/results/name/NAMEIII_timeseries.cml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,11 @@
358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down Expand Up @@ -128,7 +132,11 @@
358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer average]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down Expand Up @@ -194,7 +202,11 @@
358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down Expand Up @@ -260,7 +272,11 @@
358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down Expand Up @@ -326,7 +342,11 @@
358354.0, 358355.0, 358356.0, 358357.0]" shape="(72,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down
30 changes: 25 additions & 5 deletions lib/iris/tests/results/name/NAMEII_field.cml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@
</attributes>
<coords>
<coord>
<dimCoord bounds="[[0.0, 100.0]]" id="c87e380b" long_name="height above ground level" points="[50.0]" shape="(1,)" standard_name="height" units="Unit('m')" value_type="float64"/>
<dimCoord bounds="[[0.0, 100.0]]" id="1a539915" long_name="height above ground level" points="[50.0]" shape="(1,)" standard_name="height" units="Unit('m')" value_type="float64">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</dimCoord>
</coord>
<coord datadims="[0]">
<dimCoord bounds="[[52.36655, 52.3676555276],
Expand Down Expand Up @@ -76,7 +80,11 @@
</attributes>
<coords>
<coord>
<dimCoord bounds="[[0.0, 100.0]]" id="c87e380b" long_name="height above ground level" points="[50.0]" shape="(1,)" standard_name="height" units="Unit('m')" value_type="float64"/>
<dimCoord bounds="[[0.0, 100.0]]" id="1a539915" long_name="height above ground level" points="[50.0]" shape="(1,)" standard_name="height" units="Unit('m')" value_type="float64">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</dimCoord>
</coord>
<coord datadims="[0]">
<dimCoord bounds="[[52.36655, 52.3676555276],
Expand Down Expand Up @@ -161,7 +169,11 @@
<dimCoord bounds="[[351249.0, 351252.0]]" id="cb784457" points="[351252.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -218,7 +230,11 @@
<dimCoord bounds="[[351249.0, 351252.0]]" id="cb784457" points="[351252.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down Expand Up @@ -275,7 +291,11 @@
<dimCoord bounds="[[351249.0, 351252.0]]" id="cb784457" points="[351252.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods>
Expand Down
12 changes: 10 additions & 2 deletions lib/iris/tests/results/name/NAMEII_timeseries.cml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,11 @@
370475.0, 370476.0]" shape="(132,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down Expand Up @@ -84,7 +88,11 @@
370475.0, 370476.0]" shape="(132,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
</coord>
<coord>
<auxCoord id="457ac36d" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string"/>
<auxCoord id="e92ff7b7" long_name="z" points="[Boundary layer]" shape="(1,)" units="Unit('no_unit')" value_type="string">
<attributes>
<attribute name="positive" value="up"/>
</attributes>
</auxCoord>
</coord>
</coords>
<cellMethods/>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Copyright Iris contributors
#
# This file is part of Iris and is released under the LGPL license.
# See COPYING and COPYING.LESSER in the root of the repository for full
# licensing details.
"""
Unit tests for :func:`iris.analysis.name_loaders._build_lat_lon_for_NAME_timeseries`.
"""

# Import iris.tests first so that some things can be initialised before
# importing anything else.
import iris.tests as tests # isort:skip

from iris.fileformats.name_loaders import (
NAMECoord,
_build_lat_lon_for_NAME_timeseries,
)


class TestCellMethods(tests.IrisTest):
def test_float(self):
column_headings = {
"X": ["X = -.100 Lat-Long", "X = -1.600 Lat-Long"],
"Y": ["Y = 52.450 Lat-Long", "Y = 51. Lat-Long"],
}
lat, lon = _build_lat_lon_for_NAME_timeseries(column_headings)
self.assertIsInstance(lat, NAMECoord)
self.assertIsInstance(lon, NAMECoord)
self.assertEqual(lat.name, "latitude")
self.assertEqual(lon.name, "longitude")
self.assertIsNone(lat.dimension)
self.assertIsNone(lon.dimension)
self.assertArrayEqual(lat.values, [52.45, 51.0])
self.assertArrayEqual(lon.values, [-0.1, -1.6])

def test_int(self):
column_headings = {
"X": ["X = -1 Lat-Long", "X = -2 Lat-Long"],
"Y": ["Y = 52 Lat-Long", "Y = 51 Lat-Long"],
}
lat, lon = _build_lat_lon_for_NAME_timeseries(column_headings)
self.assertIsInstance(lat, NAMECoord)
self.assertIsInstance(lon, NAMECoord)
self.assertEqual(lat.name, "latitude")
self.assertEqual(lon.name, "longitude")
self.assertIsNone(lat.dimension)
self.assertIsNone(lon.dimension)
self.assertArrayEqual(lat.values, [52.0, 51.0])
self.assertArrayEqual(lon.values, [-1.0, -2.0])
self.assertIsInstance(lat.values[0], float)
self.assertIsInstance(lon.values[0], float)
Loading

0 comments on commit ab1e699

Please sign in to comment.