Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bring more consistency in the Platform group across sensors on conversion #1058

Merged
merged 13 commits into from
Jun 9, 2023
Merged

Bring more consistency in the Platform group across sensors on conversion #1058

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
4a4ebd2
Update convention yaml with corrected long_name entries and adding pl…
emiliom Jun 7, 2023
0b0caa9
EK80 Platform updates: cosmetic changes for clarity and consistency; …
emiliom Jun 7, 2023
9269cbb
EK60 Platform updates: ensured platform_* global attributes are inser…
emiliom Jun 7, 2023
b7e0a6f
AZFP Platform updates: added three coordinates; added many missing va…
emiliom Jun 7, 2023
40065b8
Remove from AZFP Platform test check the variables that are no longer…
emiliom Jun 8, 2023
8aa4d9f
When azfp tilt_x and tilt_y are all nan, create single-valued arrays
emiliom Jun 8, 2023
6997f14
Update echopype/convert/set_groups_azfp.py
emiliom Jun 8, 2023
064d0cd
For EK60 Platform, removed incorrect channel dimension from several v…
emiliom Jun 9, 2023
ddf0a82
Removed time3 from Platform and removed it from all variables using i…
emiliom Jun 9, 2023
be8c40d
In AZFP Platform group, change tilt units to arc_degree and set time1…
emiliom Jun 9, 2023
439819b
Merge branch 'common-platform-vars' of github.com:emiliom/echopype in…
emiliom Jun 9, 2023
be26382
Update ek80 convert tests to reflect removal of Platform time3; and a…
emiliom Jun 9, 2023
304e83f
Update echopype/tests/convert/test_convert_ek80.py
emiliom Jun 9, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 7 additions & 23 deletions echopype/convert/set_groups_azfp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,13 +139,10 @@ def set_sonar(self) -> xr.Dataset:

def set_platform(self) -> xr.Dataset:
"""Set the Platform group."""
platform_dict = {
"platform_name": self.ui_param["platform_name"],
"platform_type": self.ui_param["platform_type"],
"platform_code_ICES": self.ui_param["platform_code_ICES"],
}
platform_dict = {"platform_name": "", "platform_type": "", "platform_code_ICES": ""}
unpacked_data = self.parser_obj.unpacked_data
time2 = self.parser_obj.ping_time
time1 = [time2[0]]

# If tilt_x and/or tilt_y are all nan, create single-value time2 dimension
# and single-value (np.nan) tilt_x and tilt_y
Expand Down Expand Up @@ -182,24 +179,24 @@ def set_platform(self) -> xr.Dataset:
self._varattrs["platform_var_default"]["vertical_offset"],
),
"water_level": (
["time3"],
[np.nan],
[],
np.nan,
self._varattrs["platform_var_default"]["water_level"],
),
"tilt_x": (
["time2"],
tilt_x,
{
"long_name": "Tilt X",
"units": "degree",
"units": "arc_degree",
},
),
"tilt_y": (
["time2"],
tilt_y,
{
"long_name": "Tilt Y",
"units": "degree",
"units": "arc_degree",
},
),
**{
Expand Down Expand Up @@ -248,7 +245,7 @@ def set_platform(self) -> xr.Dataset:
"time1": (
["time1"],
# xarray and probably CF don't accept time coordinate variable with Nan values
[time2[0]],
time1,
{
**self._varattrs["platform_coord_default"]["time1"],
"comment": "Time coordinate corresponding to NMEA position data.",
Expand All @@ -265,19 +262,6 @@ def set_platform(self) -> xr.Dataset:
"orientation data.",
},
),
"time3": (
["time3"],
# xarray and probably CF don't accept time coordinate variable with Nan values
[time2[0]],
{
"axis": "T",
"long_name": "Timestamps for platform-related sampling environment",
"standard_name": "time",
"comment": "Time coordinate corresponding to platform-related "
"sampling environment. Note that Platform.time3 is "
"the same as Environment.time1.",
},
),
},
)
ds = ds.assign_attrs(platform_dict)
Expand Down
133 changes: 55 additions & 78 deletions echopype/convert/set_groups_ek60.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,12 +240,11 @@ def set_platform(self) -> xr.Dataset:
# Read lat/long from NMEA datagram
time1, msg_type, lat, lon = self._extract_NMEA_latlon()

# NMEA dataset: variables filled with nan if do not exist
platform_dict = {
"platform_name": self.ui_param["platform_name"],
"platform_type": self.ui_param["platform_type"],
"platform_code_ICES": self.ui_param["platform_code_ICES"],
}
# NMEA dataset: variables filled with np.nan if they do not exist
platform_dict = {"platform_name": "", "platform_type": "", "platform_code_ICES": ""}
# Values for the variables below having a channel (ch) dependence
# are identical across channels
ch = list(self.sorted_channel.keys())[0]
ds = xr.Dataset(
{
"latitude": (
Expand All @@ -263,6 +262,41 @@ def set_platform(self) -> xr.Dataset:
msg_type,
self._varattrs["platform_var_default"]["sentence_type"],
),
"pitch": (
["time2"],
self.parser_obj.ping_data_dict["pitch"][ch],
self._varattrs["platform_var_default"]["pitch"],
),
"roll": (
["time2"],
self.parser_obj.ping_data_dict["roll"][ch],
self._varattrs["platform_var_default"]["roll"],
),
"vertical_offset": (
["time2"],
self.parser_obj.ping_data_dict["heave"][ch],
self._varattrs["platform_var_default"]["vertical_offset"],
),
"water_level": (
[],
# a scalar, assumed to be a constant in the source transducer_depth data
self.parser_obj.ping_data_dict["transducer_depth"][ch][0],
self._varattrs["platform_var_default"]["water_level"],
),
**{
var: ([], np.nan, self._varattrs["platform_var_default"][var])
for var in [
"MRU_offset_x",
"MRU_offset_y",
"MRU_offset_z",
"MRU_rotation_x",
"MRU_rotation_y",
"MRU_rotation_z",
"position_offset_x",
"position_offset_y",
"position_offset_z",
]
},
},
coords={
"time1": (
Expand All @@ -272,43 +306,26 @@ def set_platform(self) -> xr.Dataset:
**self._varattrs["platform_coord_default"]["time1"],
"comment": "Time coordinate corresponding to NMEA position data.",
},
)
),
"time2": (
["time2"],
self.parser_obj.ping_time[ch],
{
"axis": "T",
"long_name": "Timestamps for platform motion and orientation data",
"standard_name": "time",
"comment": "Time coordinate corresponding to platform motion and "
"orientation data.",
},
),
},
)

# TODO: consider allow users to set water_level like in EK80?
# if self.ui_param['water_level'] is not None:
# water_level = self.ui_param['water_level']
# else:
# water_level = np.nan
# print('WARNING: The water_level_draft was not in the file. Value '
# 'set to None.')

# Loop over channels and merge all
ds_plat = []
for ch in self.sorted_channel.keys():
ds_tmp = xr.Dataset(
{
"pitch": (
["time2"],
self.parser_obj.ping_data_dict["pitch"][ch],
self._varattrs["platform_var_default"]["pitch"],
),
"roll": (
["time2"],
self.parser_obj.ping_data_dict["roll"][ch],
self._varattrs["platform_var_default"]["roll"],
),
"vertical_offset": (
["time2"],
self.parser_obj.ping_data_dict["heave"][ch],
self._varattrs["platform_var_default"]["vertical_offset"],
),
"water_level": (
["time3"],
self.parser_obj.ping_data_dict["transducer_depth"][ch],
self._varattrs["platform_var_default"]["water_level"],
),
"transducer_offset_x": (
[],
self.parser_obj.config_datagram["transceivers"][ch].get("pos_x", np.nan),
Expand All @@ -324,53 +341,10 @@ def set_platform(self) -> xr.Dataset:
self.parser_obj.config_datagram["transceivers"][ch].get("pos_z", np.nan),
self._varattrs["platform_var_default"]["transducer_offset_z"],
),
**{
var: ([], np.nan, self._varattrs["platform_var_default"][var])
for var in [
"MRU_offset_x",
"MRU_offset_y",
"MRU_offset_z",
"MRU_rotation_x",
"MRU_rotation_y",
"MRU_rotation_z",
"position_offset_x",
"position_offset_y",
"position_offset_z",
]
},
},
coords={
"time2": (
["time2"],
self.parser_obj.ping_time[ch],
{
"axis": "T",
"long_name": "Timestamps for platform motion and orientation data",
"standard_name": "time",
"comment": "Time coordinate corresponding to platform motion and "
"orientation data.",
},
),
"time3": (
["time3"],
self.parser_obj.ping_time[ch],
{
"axis": "T",
"long_name": "Timestamps for platform-related sampling environment",
"standard_name": "time",
"comment": "Time coordinate corresponding to platform-related "
"sampling environment.",
},
),
},
)

# Attach channel dimension/coordinate
ds_tmp = ds_tmp.expand_dims({"channel": [self.sorted_channel[ch]]})
ds_tmp["channel"] = ds_tmp["channel"].assign_attrs(
self._varattrs["beam_coord_default"]["channel"]
)

ds_tmp["frequency_nominal"] = (
["channel"],
[self.parser_obj.config_datagram["transceivers"][ch]["frequency"]],
Expand All @@ -389,6 +363,9 @@ def set_platform(self) -> xr.Dataset:
# pitch/roll/heave are the same for all freq channels
# consider only saving those from the first channel
ds_plat = xr.merge(ds_plat)
ds_plat["channel"] = ds_plat["channel"].assign_attrs(
self._varattrs["beam_coord_default"]["channel"]
)

# Merge with NMEA data
ds = xr.merge([ds, ds_plat], combine_attrs="override")
Expand Down
52 changes: 14 additions & 38 deletions echopype/convert/set_groups_ek80.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,15 +318,16 @@ def set_platform(self) -> xr.Dataset:
time2 = np.array(time2) if time2 is not None else [np.nan]

# Assemble variables into a dataset: variables filled with nan if do not exist
platform_dict = {
"platform_name": self.ui_param["platform_name"],
"platform_type": self.ui_param["platform_type"],
"platform_code_ICES": self.ui_param["platform_code_ICES"],
}
platform_dict = {"platform_name": "", "platform_type": "", "platform_code_ICES": ""}
ds = xr.Dataset(
{
"latitude": (["time1"], lat, self._varattrs["platform_var_default"]["latitude"]),
"longitude": (["time1"], lon, self._varattrs["platform_var_default"]["longitude"]),
"sentence_type": (
["time1"],
msg_type,
self._varattrs["platform_var_default"]["sentence_type"],
),
"pitch": (
["time2"],
np.array(self.parser_obj.mru.get("pitch", [np.nan])),
Expand All @@ -343,32 +344,21 @@ def set_platform(self) -> xr.Dataset:
self._varattrs["platform_var_default"]["vertical_offset"],
),
"water_level": (
["time3"],
[water_level],
[],
water_level,
self._varattrs["platform_var_default"]["water_level"],
),
"sentence_type": (
["time1"],
msg_type,
self._varattrs["platform_var_default"]["sentence_type"],
),
"drop_keel_offset": (
["time3"],
[self.parser_obj.environment["drop_keel_offset"]]
if hasattr(self.parser_obj.environment, "drop_keel_offset")
else [np.nan],
[],
self.parser_obj.environment.get("drop_keel_offset", np.nan),
),
"drop_keel_offset_is_manual": (
["time3"],
[self.parser_obj.environment["drop_keel_offset_is_manual"]]
if "drop_keel_offset_is_manual" in self.parser_obj.environment
else [np.nan],
[],
self.parser_obj.environment.get("drop_keel_offset_is_manual", np.nan),
),
"water_level_draft_is_manual": (
["time3"],
[self.parser_obj.environment["water_level_draft_is_manual"]]
if "water_level_draft_is_manual" in self.parser_obj.environment
else [np.nan],
[],
self.parser_obj.environment.get("water_level_draft_is_manual", np.nan),
),
"transducer_offset_x": (
["channel"],
Expand Down Expand Up @@ -450,20 +440,6 @@ def set_platform(self) -> xr.Dataset:
"orientation data.",
},
),
"time3": (
["time3"],
[self.parser_obj.environment["timestamp"]]
if "timestamp" in self.parser_obj.environment
else np.datetime64("NaT"),
{
"axis": "T",
"long_name": "Timestamps for platform-related sampling environment",
"standard_name": "time",
"comment": "Time coordinate corresponding to platform-related "
"sampling environment. Note that Platform.time3 is "
"the same as Environment.time1.",
},
),
},
)
ds = ds.assign_attrs(platform_dict)
Expand Down
10 changes: 5 additions & 5 deletions echopype/tests/convert/test_convert_ek80.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,23 +58,23 @@ def check_env_xml(echodata):
assert "sound_velocity_profile_depth"
assert np.array_equal(echodata["Environment"]["sound_velocity_profile_depth"], [1, 1000])

# check plat vars
# check a subset of plat vars
emiliom marked this conversation as resolved.
Show resolved Hide resolved
plat_vars = {
"drop_keel_offset": [np.nan],
"drop_keel_offset": [np.nan, 0, 7.5],
"drop_keel_offset_is_manual": [0, 1],
"water_level": [0],
"water_level_draft_is_manual": [0, 1]
emiliom marked this conversation as resolved.
Show resolved Hide resolved
}
for plat_var, expected_plat_var_values in plat_vars.items():
assert plat_var in echodata["Platform"]
assert echodata["Platform"][plat_var].dims == ("time3",)
if np.isnan(expected_plat_var_values).all():
assert np.isnan(echodata["Platform"][plat_var]).all()
else:
assert all([env_var_value in expected_plat_var_values for env_var_value in echodata["Platform"][plat_var]])
assert echodata["Platform"][plat_var] in expected_plat_var_values

# check plat dims
assert "time3" in echodata["Platform"]
assert "time1" in echodata["Platform"]
assert "time2" in echodata["Platform"]


def test_convert(ek80_new_file, dump_output_dir):
Expand Down