diff --git a/echopype/calibrate/api.py b/echopype/calibrate/api.py index 01fa7630a..c2c34c137 100644 --- a/echopype/calibrate/api.py +++ b/echopype/calibrate/api.py @@ -153,7 +153,7 @@ def compute_Sv(echodata: EchoData, **kwargs) -> xr.Dataset: - for EK60 echosounder, allowed parameters include: `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"` - for AZFP echosounder, allowed parameters include: - `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"` + `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"` Passing in calibration parameters for other echosounders are not currently supported. @@ -242,7 +242,7 @@ def compute_TS(echodata: EchoData, **kwargs): - for EK60 echosounder, allowed parameters include: `"sa_correction"`, `"gain_correction"`, `"equivalent_beam_angle"` - for AZFP echosounder, allowed parameters include: - `"EL"`, `"DS"`, `"TVR"`, `"VTX"`, `"equivalent_beam_angle"`, `"Sv_offset"` + `"EL"`, `"DS"`, `"TVR"`, `"VTX0"`, `"equivalent_beam_angle"`, `"Sv_offset"` Passing in calibration parameters for other echosounders are not currently supported. diff --git a/echopype/calibrate/cal_params.py b/echopype/calibrate/cal_params.py index 0c861c9e9..5d020cc9b 100644 --- a/echopype/calibrate/cal_params.py +++ b/echopype/calibrate/cal_params.py @@ -29,7 +29,7 @@ "impedance_transceiver", # z_er "receiver_sampling_frequency", ), - "AZFP": ("EL", "DS", "TVR", "VTX", "equivalent_beam_angle", "Sv_offset"), + "AZFP": ("EL", "DS", "TVR", "VTX0", "equivalent_beam_angle", "Sv_offset"), } EK80_DEFAULT_PARAMS = { @@ -352,7 +352,7 @@ def get_cal_params_AZFP(beam: xr.DataArray, vend: xr.DataArray, user_dict: dict) out_dict[p] = beam[p] # has only channel dim # Params from Vendor_specific group - elif p in ["EL", "DS", "TVR", "VTX", "Sv_offset"]: + elif p in ["EL", "DS", "TVR", "VTX0", "Sv_offset"]: out_dict[p] = vend[p] # these params only have the channel dimension return out_dict diff --git a/echopype/calibrate/calibrate_azfp.py b/echopype/calibrate/calibrate_azfp.py index bc2f848f0..1299be8bb 100644 --- a/echopype/calibrate/calibrate_azfp.py +++ b/echopype/calibrate/calibrate_azfp.py @@ -63,7 +63,7 @@ def _cal_power_samples(self, cal_type, **kwargs): # TODO: take care of dividing by zero encountered in log10 spreading_loss = 20 * np.log10(self.range_meter) absorption_loss = 2 * self.env_params["sound_absorption"] * self.range_meter - SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX"]) # eq.(2) + SL = self.cal_params["TVR"] + 20 * np.log10(self.cal_params["VTX0"]) # eq.(2) # scaling factor (slope) in Fig.G-1, units Volts/dB], see p.84 a = self.cal_params["DS"] diff --git a/echopype/calibrate/range.py b/echopype/calibrate/range.py index aa93a6061..b3c049c5f 100644 --- a/echopype/calibrate/range.py +++ b/echopype/calibrate/range.py @@ -60,7 +60,7 @@ def compute_range_AZFP(echodata: EchoData, env_params: Dict, cal_type: str) -> x # Notation below follows p.86 of user manual N = vend["number_of_samples_per_average_bin"] # samples per bin f = vend["digitization_rate"] # digitization rate - L = vend["lockout_index"] # number of lockout samples + L = vend["lock_out_index"] # number of lockout samples # keep this in ref of AZFP matlab code, # set to 1 since we want to calculate from raw data diff --git a/echopype/convert/parse_azfp.py b/echopype/convert/parse_azfp.py index a890068a7..f1ef32bb6 100644 --- a/echopype/convert/parse_azfp.py +++ b/echopype/convert/parse_azfp.py @@ -1,5 +1,5 @@ import os -import xml.dom.minidom +import xml.etree.ElementTree as ET from collections import defaultdict from datetime import datetime as dt from struct import unpack @@ -8,48 +8,11 @@ import numpy as np from ..utils.log import _init_logger +from ..utils.misc import camelcase2snakecase from .parse_base import ParseBase FILENAME_DATETIME_AZFP = "\\w+.01A" -XML_INT_PARAMS = { - "NumFreq": "num_freq", - "SerialNumber": "serial_number", - "BurstInterval": "burst_interval", - "PingsPerBurst": "pings_per_burst", - "AverageBurstPings": "average_burst_pings", - "SensorsFlag": "sensors_flag", -} -XML_FLOAT_PARAMS = [ - # Temperature coeffs - "ka", - "kb", - "kc", - "A", - "B", - "C", - # Tilt coeffs - "X_a", - "X_b", - "X_c", - "X_d", - "Y_a", - "Y_b", - "Y_c", - "Y_d", -] -XML_FREQ_PARAMS = { - "RangeSamples": "range_samples", - "RangeAveragingSamples": "range_averaging_samples", - "DigRate": "dig_rate", - "LockOutIndex": "lockout_index", - "Gain": "gain", - "PulseLen": "pulse_length", - "DS": "DS", - "EL": "EL", - "TVR": "TVR", - "VTX0": "VTX", - "BP": "BP", -} + HEADER_FIELDS = ( ("profile_flag", "u2"), ("profile_number", "u2"), @@ -64,7 +27,7 @@ ("second", "u2"), # Second ("hundredths", "u2"), # Hundredths of a second ("dig_rate", "u2", 4), # Digitalization rate for each channel - ("lockout_index", "u2", 4), # Lockout index for each channel + ("lock_out_index", "u2", 4), # Lockout index for each channel ("num_bins", "u2", 4), # Number of bins for each channel ( "range_samples_per_bin", @@ -88,7 +51,7 @@ ("num_chan", "u1"), # 1, 2, 3, or 4 ("gain", "u1", 4), # gain channel 1-4 ("spare_chan", "u1"), # spare channel - ("pulse_length", "u2", 4), # Pulse length chan 1-4 uS + ("pulse_len", "u2", 4), # Pulse length chan 1-4 uS ("board_num", "u2", 4), # The board the data came from channel 1-4 ("frequency", "u2", 4), # frequency for channel 1-4 in kHz ( @@ -118,34 +81,41 @@ def __init__(self, file, params, storage_options={}, dgram_zarr_vars={}): self.xml_path = params # Class attributes - self.parameters = dict() + self.parameters = defaultdict(list) self.unpacked_data = defaultdict(list) self.sonar_type = "AZFP" def load_AZFP_xml(self): - """Parse XML file to get params for reading AZFP data.""" - """Parses the AZFP XML file. """ - - def get_value_by_tag_name(tag_name, element=0): - """Returns the value in an XML tag given the tag name and the number of occurrences.""" - return px.getElementsByTagName(tag_name)[element].childNodes[0].data + Parses the AZFP XML file. + """ xmlmap = fsspec.get_mapper(self.xml_path, **self.storage_options) - px = xml.dom.minidom.parse(xmlmap.fs.open(xmlmap.root)) - - # Retrieve integer parameters from the xml file - for old_name, new_name in XML_INT_PARAMS.items(): - self.parameters[new_name] = int(get_value_by_tag_name(old_name)) - # Retrieve floating point parameters from the xml file - for param in XML_FLOAT_PARAMS: - self.parameters[param] = float(get_value_by_tag_name(param)) - # Retrieve frequency dependent parameters from the xml file - for old_name, new_name in XML_FREQ_PARAMS.items(): - self.parameters[new_name] = [ - float(get_value_by_tag_name(old_name, ch)) - for ch in range(self.parameters["num_freq"]) - ] + root = ET.parse(xmlmap.fs.open(xmlmap.root)).getroot() + + for child in root.iter(): + if len(child.tag) > 3 and not child.tag.startswith("VTX"): + camel_case_tag = camelcase2snakecase(child.tag) + else: + camel_case_tag = child.tag + if len(child.attrib) > 0: + for key, val in child.attrib.items(): + self.parameters[camel_case_tag + "_" + camelcase2snakecase(key)].append(val) + + if all(char == "\n" for char in child.text): + continue + else: + try: + val = int(child.text) + except ValueError: + val = float(child.text) + + self.parameters[camel_case_tag].append(val) + + # Handling the case where there is only one value for each parameter + for key, val in self.parameters.items(): + if len(val) == 1: + self.parameters[key] = val[0] def _compute_temperature(self, ping_num, is_valid): """ @@ -245,7 +215,6 @@ def _test_valid_params(params): header_chunk = file.read(self.HEADER_SIZE) if header_chunk: header_unpacked = unpack(self.HEADER_FORMAT, header_chunk) - # Reading will stop if the file contains an unexpected flag if self._split_header(file, header_unpacked): # Appends the actual 'data values' to unpacked_data @@ -354,12 +323,12 @@ def _split_header(self, raw, header_unpacked): field_w_freq = ( "dig_rate", - "lockout_index", + "lock_out_index", "num_bins", "range_samples_per_bin", # fields with num_freq data "data_type", "gain", - "pulse_length", + "pulse_len", "board_num", "frequency", ) @@ -417,12 +386,12 @@ def _check_uniqueness(self): # fields with num_freq data field_w_freq = ( "dig_rate", - "lockout_index", + "lock_out_index", "num_bins", "range_samples_per_bin", "data_type", "gain", - "pulse_length", + "pulse_len", "board_num", "frequency", ) @@ -478,22 +447,22 @@ def _get_ping_time(self): self.ping_time = ping_time @staticmethod - def _calc_Sv_offset(f, pulse_length): + def _calc_Sv_offset(f, pulse_len): """Calculate the compensation factor for Sv calculation.""" # TODO: this method seems should be in echopype.process if f > 38000: - if pulse_length == 300: + if pulse_len == 300: return 1.1 - elif pulse_length == 500: + elif pulse_len == 500: return 0.8 - elif pulse_length == 700: + elif pulse_len == 700: return 0.5 - elif pulse_length == 900: + elif pulse_len == 900: return 0.3 - elif pulse_length == 1000: + elif pulse_len == 1000: return 0.3 else: - if pulse_length == 500: + if pulse_len == 500: return 1.1 - elif pulse_length == 1000: + elif pulse_len == 1000: return 0.7 diff --git a/echopype/convert/set_groups_azfp.py b/echopype/convert/set_groups_azfp.py index 57f754ce9..9c37eec1e 100644 --- a/echopype/convert/set_groups_azfp.py +++ b/echopype/convert/set_groups_azfp.py @@ -81,13 +81,14 @@ def _create_unique_channel_name(self): """ serial_number = self.parser_obj.unpacked_data["serial_number"] + frequency_number = self.parser_obj.parameters["frequency_number"] if serial_number.size == 1: freq_as_str = self.freq_sorted.astype(int).astype(str) # TODO: replace str(i+1) with Frequency Number from XML channel_id = [ - str(serial_number) + "-" + freq + "-" + str(i + 1) + str(serial_number) + "-" + freq + "-" + frequency_number[i] for i, freq in enumerate(freq_as_str) ] @@ -146,8 +147,7 @@ def set_sonar(self) -> xr.Dataset: "sonar_model": self.sonar_model, "sonar_serial_number": int(self.parser_obj.unpacked_data["serial_number"]), "sonar_software_name": "AZFP", - # TODO: software version is hardwired. Read it from the XML file's AZFP_Version node - "sonar_software_version": "1.4", + "sonar_software_version": "based on AZFP Matlab version 1.4", "sonar_type": "echosounder", } ds = ds.assign_attrs(sonar_attr_dict) @@ -320,7 +320,7 @@ def set_beam(self) -> List[xr.Dataset]: del N_tmp tdn = ( - unpacked_data["pulse_length"][self.freq_ind_sorted] / 1e6 + unpacked_data["pulse_len"][self.freq_ind_sorted] / 1e6 ) # Convert microseconds to seconds range_samples_per_bin = unpacked_data["range_samples_per_bin"][ self.freq_ind_sorted @@ -500,7 +500,7 @@ def set_vendor(self) -> xr.Dataset: unpacked_data = self.parser_obj.unpacked_data parameters = self.parser_obj.parameters ping_time = self.parser_obj.ping_time - tdn = parameters["pulse_length"][self.freq_ind_sorted] / 1e6 + tdn = parameters["pulse_len"][self.freq_ind_sorted] / 1e6 anc = np.array(unpacked_data["ancillary"]) # convert to np array for easy slicing # Build variables in the output xarray Dataset @@ -508,7 +508,7 @@ def set_vendor(self) -> xr.Dataset: for ind, ich in enumerate(self.freq_ind_sorted): # TODO: should not access the private function, better to compute Sv_offset in parser Sv_offset[ind] = self.parser_obj._calc_Sv_offset( - self.freq_sorted[ind], unpacked_data["pulse_length"][ich] + self.freq_sorted[ind], unpacked_data["pulse_len"][ich] ) ds = xr.Dataset( @@ -532,9 +532,9 @@ def set_vendor(self) -> xr.Dataset: "A/D converter when digitizing the returned acoustic signal" }, ), - "lockout_index": ( + "lock_out_index": ( ["channel"], - unpacked_data["lockout_index"][self.freq_ind_sorted], + unpacked_data["lock_out_index"][self.freq_ind_sorted], { "long_name": "The distance, rounded to the nearest Bin Size after the " "pulse is transmitted that over which AZFP will ignore echoes" @@ -648,6 +648,17 @@ def set_vendor(self) -> xr.Dataset: parameters["gain"][self.freq_ind_sorted], {"long_name": "(From XML file) Gain correction"}, ), + "instrument_type": parameters["instrument_type"][0], + "minor": parameters["minor"], + "major": parameters["major"], + "date": parameters["date"], + "program": parameters["program"], + "cpu": parameters["cpu"], + "serial_number": parameters["serial_number"], + "board_version": parameters["board_version"], + "file_version": parameters["file_version"], + "parameter_version": parameters["parameter_version"], + "configuration_version": parameters["configuration_version"], "XML_digitization_rate": ( ["channel"], parameters["dig_rate"][self.freq_ind_sorted], @@ -659,7 +670,7 @@ def set_vendor(self) -> xr.Dataset: ), "XML_lockout_index": ( ["channel"], - parameters["lockout_index"][self.freq_ind_sorted], + parameters["lock_out_index"][self.freq_ind_sorted], { "long_name": "(From XML file) The distance, rounded to the nearest " "Bin Size after the pulse is transmitted that over which AZFP will " @@ -680,10 +691,25 @@ def set_vendor(self) -> xr.Dataset: "units": "dB re 1uPa/V at 1m", }, ), - "VTX": ( + "VTX0": ( ["channel"], - parameters["VTX"][self.freq_ind_sorted], - {"long_name": "Amplified voltage sent to the transducer"}, + parameters["VTX0"][self.freq_ind_sorted], + {"long_name": "Amplified voltage 0 sent to the transducer"}, + ), + "VTX1": ( + ["channel"], + parameters["VTX1"][self.freq_ind_sorted], + {"long_name": "Amplified voltage 1 sent to the transducer"}, + ), + "VTX2": ( + ["channel"], + parameters["VTX2"][self.freq_ind_sorted], + {"long_name": "Amplified voltage 2 sent to the transducer"}, + ), + "VTX3": ( + ["channel"], + parameters["VTX3"][self.freq_ind_sorted], + {"long_name": "Amplified voltage 3 sent to the transducer"}, ), "Sv_offset": (["channel"], Sv_offset), "number_of_samples_digitized_per_pings": ( diff --git a/echopype/convert/utils/ek_raw_parsers.py b/echopype/convert/utils/ek_raw_parsers.py index dae9161a7..8c349a0f2 100644 --- a/echopype/convert/utils/ek_raw_parsers.py +++ b/echopype/convert/utils/ek_raw_parsers.py @@ -16,6 +16,7 @@ import numpy as np from ...utils.log import _init_logger +from ...utils.misc import camelcase2snakecase from .ek_date_conversion import nt_to_unix TCVR_CH_NUM_MATCHER = re.compile(r"\d{6}-\w{1,2}|\w{12}-\w{1,2}") @@ -706,22 +707,6 @@ def _unpack_contents(self, raw_string, bytes_read, version): :returns: None """ - def from_CamelCase(xml_param): - """ - convert name from CamelCase to fit with existing naming convention by - inserting an underscore before each capital and then lowering the caps - e.g. CamelCase becomes camel_case. - """ - idx = list(reversed([i for i, c in enumerate(xml_param) if c.isupper()])) - param_len = len(xml_param) - for i in idx: - # check if we should insert an underscore - if i > 0 and i < param_len: - xml_param = xml_param[:i] + "_" + xml_param[i:] - xml_param = xml_param.lower() - - return xml_param - def dict_to_dict(xml_dict, data_dict, parse_opts): """ dict_to_dict appends the ETree xml value dicts to a provided dictionary @@ -760,13 +745,13 @@ def dict_to_dict(xml_dict, data_dict, parse_opts): data_dict[parse_opts[k][1]] = data else: # add using the default key name wrangling - data_dict[from_CamelCase(k)] = data + data_dict[camelcase2snakecase(k)] = data else: # nothing to do with the value string data = xml_dict[k] # add the parameter to the provided dictionary - data_dict[from_CamelCase(k)] = data + data_dict[camelcase2snakecase(k)] = data header_values = struct.unpack( self.header_fmt(version), raw_string[: self.header_size(version)] diff --git a/echopype/tests/convert/test_convert_azfp.py b/echopype/tests/convert/test_convert_azfp.py index 075fde0fb..4b8f411b6 100644 --- a/echopype/tests/convert/test_convert_azfp.py +++ b/echopype/tests/convert/test_convert_azfp.py @@ -10,12 +10,14 @@ from scipy.io import loadmat from echopype import open_raw import pytest +from echopype.convert.parse_azfp import ParseAZFP @pytest.fixture def azfp_path(test_path): return test_path["AZFP"] + def check_platform_required_scalar_vars(echodata): # check convention-required variables in the Platform group for var in [ @@ -172,3 +174,53 @@ def test_convert_azfp_01a_notemperature_notilt(azfp_path): assert "tilt_y" in echodata["Platform"] assert echodata["Platform"]["tilt_x"].isnull().all() assert echodata["Platform"]["tilt_y"].isnull().all() + + +def test_load_parse_azfp_xml(azfp_path): + + azfp_01a_path = azfp_path / '17082117.01A' + azfp_xml_path = azfp_path / '17030815.XML' + parseAZFP = ParseAZFP(str(azfp_01a_path), str(azfp_xml_path)) + parseAZFP.load_AZFP_xml() + expected_params = ['instrument_type_string', 'instrument_type', 'major', 'minor', 'date', + 'program_name', 'program', 'CPU', 'serial_number', 'board_version', + 'file_version', 'parameter_version', 'configuration_version', 'eclock', + 'digital_board_version', 'sensors_flag_pressure_sensor_installed', + 'sensors_flag_paros_installed', 'sensors_flag', 'U0', 'Y1', 'Y2', 'Y3', + 'C1', 'C2', 'C3', 'D1', 'D2', 'T1', 'T2', 'T3', 'T4', 'T5', 'X_a', 'X_b', + 'X_c', 'X_d', 'Y_a', 'Y_b', 'Y_c', 'Y_d', 'period', 'ppm_offset', + 'calibration', 'a0', 'a1', 'a2', 'a3', 'ka', 'kb', 'kc', 'A', 'B', 'C', + 'num_freq', 'kHz_units', 'kHz', 'TVR', 'num_vtx', 'VTX0', 'VTX1', 'VTX2', + 'VTX3', 'BP', 'EL', 'DS', 'min_pulse_len', 'sound_speed', + 'start_date_svalue', 'start_date', 'num_frequencies', 'num_phases', + 'data_output_svalue', 'data_output', 'frequency_units', 'frequency', + 'phase_number', 'phase_type_svalue', 'phase_type', 'duration_svalue', + 'duration', 'ping_period_units', 'ping_period', 'burst_interval_units', + 'burst_interval', 'pings_per_burst_units', 'pings_per_burst', + 'average_burst_pings_units', 'average_burst_pings', 'frequency_number', + 'acquire_frequency_units', 'acquire_frequency', 'pulse_len_units', + 'pulse_len', 'dig_rate_units', 'dig_rate', 'range_samples_units', + 'range_samples', 'range_averaging_samples_units', 'range_averaging_samples', + 'lock_out_index_units', 'lock_out_index', 'gain_units', 'gain', + 'storage_format_units', 'storage_format'] + assert set(parseAZFP.parameters.keys()) == set(expected_params) + assert list(set(parseAZFP.parameters['instrument_type_string']))[0] == 'AZFP' + assert isinstance(parseAZFP.parameters['num_freq'], int) + assert isinstance(parseAZFP.parameters['pulse_len'], list) + assert parseAZFP.parameters['num_freq'] == 4 + assert len(parseAZFP.parameters['frequency_number']) == 4 + assert parseAZFP.parameters['frequency_number'] == ['1', '2', '3', '4'] + assert parseAZFP.parameters['kHz'] == [125, 200, 455, 769] + + expected_len_params = ['acquire_frequency', 'pulse_len', 'dig_rate', 'range_samples', + 'range_averaging_samples', 'lock_out_index', 'gain', 'storage_format'] + assert all(len(parseAZFP.parameters[x]) == 4 for x in expected_len_params) + assert parseAZFP.parameters['acquire_frequency'] == [1, 1, 1, 1] + assert parseAZFP.parameters['pulse_len'] == [300, 300, 300, 300] + assert parseAZFP.parameters['dig_rate'] == [20000, 20000, 20000, 20000] + assert parseAZFP.parameters['range_samples'] == [1752, 1752, 1764, 540] + assert parseAZFP.parameters['range_averaging_samples'] == [4, 4, 4, 4] + assert parseAZFP.parameters['lock_out_index'] == [0, 0, 0, 0] + assert parseAZFP.parameters['gain'] == [1, 1, 1, 1] + assert parseAZFP.parameters['storage_format'] == [1, 1, 1, 1] + diff --git a/echopype/utils/misc.py b/echopype/utils/misc.py new file mode 100644 index 000000000..e6cb8fb11 --- /dev/null +++ b/echopype/utils/misc.py @@ -0,0 +1,13 @@ +def camelcase2snakecase(camel_case_str): + """ + Convert string from CamelCase to snake_case + e.g. CamelCase becomes camel_case. + """ + idx = list(reversed([i for i, c in enumerate(camel_case_str) if c.isupper()])) + param_len = len(camel_case_str) + for i in idx: + # check if we should insert an underscore + if i > 0 and i < param_len: + camel_case_str = camel_case_str[:i] + "_" + camel_case_str[i:] + + return camel_case_str.lower()