Skip to content

Commit

Permalink
Additional test and related fixes stemming from Platform group PR OSO…
Browse files Browse the repository at this point in the history 
…ceanAcoustics#1058 [all tests ci] (OSOceanAcoustics#1061)

* Fix processing level L1A test to account for AZFP open_raw now creating empty lat & lon variables

* Remove obsolete environmental parameters tests

* Fix test_plot tests and visualize api involving water_level > vertical_offset and the removal of channel dim from ek60 platform vars

* Cosmetic change simply to trigger all ci tests

* Remove remnant test case for ek60 vertical_offset channel dim; correct data-type typing hint
  • Loading branch information
@emiliom @lsetiawan
emiliom authored and lsetiawan committed Jul 28, 2023
1 parent 3693a2d commit 119cdb0
Show file tree
Hide file tree
Showing 4 changed files with 72 additions and 202 deletions.
132 changes: 0 additions & 132 deletions echopype/tests/calibrate/test_calibrate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,135 +263,3 @@ def test_compute_Sv_ek80_CW_complex_BB_complex(ek80_cal_path, ek80_path):
ed, waveform_mode="BB", encode_mode="complex"
)
assert isinstance(ds_Sv, xr.Dataset)


def test_env_params(ek60_path):
"""
Tests EnvParams interpolation
"""

ed = ep.open_raw(ek60_path / "ncei-wcsd" / "Summer2017-D20170620-T011027.raw", "EK60")

# stationary
# since the raw ping_times go up to 1:43 but the env_params ping_time stops at 1:25,
# values after 1:25 will be extrapolated
env_params_data = xr.Dataset(
data_vars={
"pressure": ("time3", np.arange(50)),
"salinity": ("time3", np.arange(50)),
"temperature": ("time3", np.arange(50)),
},
coords={
"time3": np.arange("2017-06-20T01:00", "2017-06-20T01:25", np.timedelta64(30, "s"), dtype="datetime64[ns]")
}
)
env_params = EnvParams(env_params_data, "stationary")
converted_env_params = env_params._apply(ed)
for var in converted_env_params.values():
assert np.all(np.diff(var) > 0)
assert np.all(0 <= var)
assert np.all(var < 100)
# TODO: substitute ping_time and input values of the env variables
# so that interpolation gives nice outputs
known_values = {
"temperature": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.03483333973334,
},
"salinity": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.0348333397333,
},
"pressure": {
"2017-06-20T01:10:27.136999936": 20.904566664533334,
"2017-06-20T01:10:28.149000192": 20.9383000064,
"2017-06-20T01:10:29.160999936": 20.9720333312,
"2017-06-20T01:10:30.174000128": 21.005800004266668,
"2017-06-20T01:10:31.184999936": 21.039499997866667,
"2017-06-20T01:42:56.995999744": 85.89986665813333,
"2017-06-20T01:42:58.008999936": 85.9336333312,
"2017-06-20T01:42:59.020000256": 85.96733334186666,
"2017-06-20T01:43:00.032000000": 86.00106666666667,
"2017-06-20T01:43:01.045000192": 86.03483333973334,
}
}
for var, values in known_values.items():
for time, value in values.items():
assert np.isclose(converted_env_params[var].sel(time1=time), value)

# mobile
rng = np.random.default_rng(0)
env_params_data = xr.Dataset(
data_vars={
"pressure": ("time", np.arange(100)),
"salinity": ("time", np.arange(100)),
"temperature": ("time", np.arange(100)),
},
coords={
"latitude": ("time", rng.random(size=100) + 44),
"longitude": ("time", rng.random(size=100) - 125),
}
)
env_params = EnvParams(env_params_data, "mobile")
converted_env_params = env_params._apply(ed)
for var in converted_env_params.values():
assert np.all(0 <= var[~np.isnan(var)])
assert np.all(var[~np.isnan(var)] < 100)
known_values = {
"temperature": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
"salinity": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
"pressure": {
"2017-06-20T01:10:27.136999936": np.nan,
"2017-06-20T01:10:28.149000192": 72.57071056437047,
"2017-06-20T01:10:29.160999936": 72.56164311204404,
"2017-06-20T01:10:30.174000128": 72.5641609908268,
"2017-06-20T01:10:31.184999936": 72.5540675620769,
"2017-06-20T01:42:56.995999744": 64.78639664394186,
"2017-06-20T01:42:58.008999936": 64.76543272189699,
"2017-06-20T01:42:59.020000256": 64.77890258158483,
"2017-06-20T01:43:00.032000000": 64.76186093048929,
"2017-06-20T01:43:01.045000192": 64.76763007606817,
},
}
for var, values in known_values.items():
for time, value in values.items():
print(var, time, value)
assert np.isnan(value) or np.isclose(converted_env_params[var].sel(time1=time), value)
11 changes: 9 additions & 2 deletions echopype/tests/utils/test_processinglevels_integration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,14 @@ def _absence_test(test_ds):
assert "processing_level_url" not in test_ds.attrs

# ---- Convert raw file and update_platform
def _var_presence_notnan_test(name):
if name in ed['Platform'].data_vars and not ed["Platform"][name].isnull().all():
return True
else:
return False

ed = ep.open_raw(raw_path, xml_path=xml_path, sonar_model=sonar_model)
if "longitude" in ed['Platform'].data_vars and "latitude" in ed['Platform'].data_vars:
if _var_presence_notnan_test("longitude") and _var_presence_notnan_test("latitude"):
_presence_test(ed["Top-level"], "Level 1A")
elif "longitude" in extras and "latitude" in extras:
_absence_test(ed["Top-level"])
Expand Down Expand Up @@ -120,7 +126,8 @@ def _freqdiff_applymask(test_ds):

# ---- Compute MVBS
# compute_MVBS expects a variable named "Sv"
# No product level is assigned because at present compute_MVBS drops the lat/lon data associated with the input Sv dataset
# No product level is assigned because at present compute_MVBS drops the lat/lon data
# associated with the input Sv dataset
# ds = ds.rename_vars(name_dict={"Sv": "Sv_unmasked", "Sv_ch0": "Sv"})
mvbs_ds = ep.commongrid.compute_MVBS(ds, range_meter_bin=30, ping_time_bin='1min')
_absence_test(mvbs_ds)
68 changes: 31 additions & 37 deletions echopype/tests/visualize/test_plot.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,20 +197,20 @@ def test_plot_mvbs(


@pytest.mark.parametrize(
("water_level", "expect_warning"),
("vertical_offset", "expect_warning"),
[
(True, False),
([True], True),
(False, True),
(xr.DataArray(np.array(50.0)).expand_dims({'channel': 3}), False),
(xr.DataArray(np.array(50.0)), False),
([10, 30.5], False),
(10, False),
(30.5, False),
],
)
def test_water_level_echodata(water_level, expect_warning, caplog):
def test_vertical_offset_echodata(vertical_offset, expect_warning, caplog):
from echopype.echodata import EchoData
from echopype.visualize.api import _add_water_level
from echopype.visualize.api import _add_vertical_offset
echopype.verbose()

filepath = ek60_path / "ncei-wcsd" / "Summer2017-D20170719-T211347.raw"
Expand All @@ -231,45 +231,41 @@ def test_water_level_echodata(water_level, expect_warning, caplog):

single_array = range_in_meter.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
ping_time='2017-07-19T21:13:47.984999936').values
no_input_water_level = False

if isinstance(water_level, list):
water_level = water_level[0]
echodata["Platform"] = echodata["Platform"].drop_vars('water_level')
no_input_water_level = True

if isinstance(water_level, xr.DataArray):
if 'channel' in water_level.dims:
original_array = single_array + water_level.isel(channel=0).values
elif isinstance(water_level, bool) and water_level is True:
if no_input_water_level is False:
no_input_vertical_offset = False

if isinstance(vertical_offset, list):
vertical_offset = vertical_offset[0]
echodata["Platform"] = echodata["Platform"].drop_vars('vertical_offset')
no_input_vertical_offset = True

if isinstance(vertical_offset, xr.DataArray):
original_array = single_array + vertical_offset.values
elif isinstance(vertical_offset, bool) and vertical_offset is True:
if not no_input_vertical_offset:
original_array = (
single_array
+ echodata["Platform"].water_level.sel(
channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
time3='2017-07-19T21:13:47.984999936'
).values
+ echodata["Platform"].vertical_offset.sel(time2='2017-07-19T21:13:47.984999936').values
)
else:
original_array = single_array
elif water_level is not False and isinstance(water_level, (int, float)):
original_array = single_array + water_level
elif vertical_offset is not False and isinstance(vertical_offset, (int, float)):
original_array = single_array + vertical_offset
else:
original_array = single_array

results = None
try:
results = _add_water_level(
results = _add_vertical_offset(
range_in_meter=range_in_meter,
water_level=water_level,
vertical_offset=vertical_offset,
data_type=EchoData,
platform_data=echodata["Platform"],
)
if expect_warning:
assert 'WARNING' in caplog.text
except Exception as e:
assert isinstance(e, ValueError)
assert str(e) == 'Water level must have any of these dimensions: channel, ping_time, range_sample' # noqa
assert str(e) == 'vertical_offset must have any of these dimensions: ping_time, range_sample' # noqa

if isinstance(results, xr.DataArray):
final_array = results.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
Expand All @@ -280,18 +276,17 @@ def test_water_level_echodata(water_level, expect_warning, caplog):


@pytest.mark.parametrize(
("water_level", "expect_warning"),
("vertical_offset", "expect_warning"),
[
(True, True),
(False, True),
(xr.DataArray(np.array(50.0)).expand_dims({'channel': 3}), False),
(xr.DataArray(np.array(50.0)), False),
(10, False),
(30.5, False),
],
)
def test_water_level_Sv_dataset(water_level, expect_warning, caplog):
from echopype.visualize.api import _add_water_level
def test_vertical_offset_Sv_dataset(vertical_offset, expect_warning, caplog):
from echopype.visualize.api import _add_vertical_offset
echopype.verbose()

filepath = ek60_path / "ncei-wcsd" / "Summer2017-D20170719-T211347.raw"
Expand All @@ -308,27 +303,26 @@ def test_water_level_Sv_dataset(water_level, expect_warning, caplog):
single_array = range_in_meter.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
ping_time='2017-07-19T21:13:47.984999936').values

if isinstance(water_level, xr.DataArray):
if 'channel' in water_level.dims:
original_array = single_array + water_level.isel(channel=0).values
elif not isinstance(water_level, bool) and isinstance(water_level, (int, float)):
original_array = single_array + water_level
if isinstance(vertical_offset, xr.DataArray):
original_array = single_array + vertical_offset.values
elif not isinstance(vertical_offset, bool) and isinstance(vertical_offset, (int, float)):
original_array = single_array + vertical_offset
else:
original_array = single_array

results = None
try:
results = _add_water_level(
results = _add_vertical_offset(
range_in_meter=range_in_meter,
water_level=water_level,
vertical_offset=vertical_offset,
data_type=xr.Dataset,
)

if expect_warning:
assert 'WARNING' in caplog.text
except Exception as e:
assert isinstance(e, ValueError)
assert str(e) == 'Water level must have any of these dimensions: channel, ping_time, range_sample' # noqa
assert str(e) == 'vertical_offset must have any of these dimensions: ping_time, range_sample' # noqa

if isinstance(results, xr.DataArray):
final_array = results.sel(channel='GPT 18 kHz 009072058c8d 1-1 ES18-11',
Expand Down
Loading

0 comments on commit 119cdb0

Please sign in to comment.