diff --git a/asammdf/blocks/mdf_v3.py b/asammdf/blocks/mdf_v3.py index aa7929f8f..47626c733 100644 --- a/asammdf/blocks/mdf_v3.py +++ b/asammdf/blocks/mdf_v3.py @@ -398,6 +398,8 @@ def _prepare_record(self, group): """ + byte_order = self.identification.byte_order + parents, dtypes = group.parents, group.types if parents is None: grp = group @@ -457,7 +459,7 @@ def _prepare_record(self, group): if data_type == v23c.DATA_TYPE_STRING: next_byte_aligned_position = parent_start_offset + size if next_byte_aligned_position <= record_size: - dtype_pair = (name, get_fmt_v3(data_type, size)) + dtype_pair = (name, get_fmt_v3(data_type, size, byte_order)) types.append(dtype_pair) parents[original_index] = name, bit_offset @@ -467,7 +469,7 @@ def _prepare_record(self, group): elif data_type == v23c.DATA_TYPE_BYTEARRAY: next_byte_aligned_position = parent_start_offset + size if next_byte_aligned_position <= record_size: - dtype_pair = (name, get_fmt_v3(data_type, size)) + dtype_pair = (name, get_fmt_v3(data_type, size, byte_order)) types.append(dtype_pair) parents[original_index] = name, bit_offset else: @@ -479,7 +481,10 @@ def _prepare_record(self, group): byte_size += 1 bit_size = byte_size * 8 - if data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA): + if ( + data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA) + or data_type in (v23c.DATA_TYPE_SIGNED, v23c.DATA_TYPE_UNSIGNED) and byte_order == v23c.BYTE_ORDER_MOTOROLA + ): if size > 32: next_byte_aligned_position = parent_start_offset + 64 @@ -504,7 +509,7 @@ def _prepare_record(self, group): next_byte_aligned_position = parent_start_offset + 8 if next_byte_aligned_position <= record_size: - dtype_pair = (name, get_fmt_v3(data_type, size)) + dtype_pair = (name, get_fmt_v3(data_type, size, byte_order)) types.append(dtype_pair) parents[original_index] = name, bit_offset else: @@ -522,7 +527,10 @@ def _prepare_record(self, group): max_overlapping = next_byte_aligned_position - byte_start_offset if max_overlapping >= bit_size: - if data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA): + if ( + data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA) + or data_type in (v23c.DATA_TYPE_SIGNED, v23c.DATA_TYPE_UNSIGNED) and byte_order == v23c.BYTE_ORDER_MOTOROLA + ): parents[original_index] = ( current_parent, bit_offset + max_overlapping - bit_size, @@ -640,7 +648,7 @@ def _get_not_byte_aligned_data(self, data, group, ch_nr): if data_type in v23c.SIGNED_INT: return as_non_byte_sized_signed_int(vals, bit_count) elif data_type in v23c.FLOATS: - return vals.view(get_fmt_v3(data_type, bit_count)) + return vals.view(get_fmt_v3(data_type, bit_count, self.identification.byte_order)) else: return vals @@ -1137,13 +1145,6 @@ def append( else: timestamps = array([]) - if self.version < "3.10": - if timestamps.dtype.byteorder == ">": - timestamps = timestamps.byteswap().newbyteorder() - for signal in signals: - if signal.samples.dtype.byteorder == ">": - signal.samples = signal.samples.byteswap().newbyteorder() - if self.version >= "3.00": channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE elif self.version >= "2.10": @@ -1985,13 +1986,6 @@ def _append_dataframe(self, df, source_info="", units=None): timestamps = t - # if self.version < '3.10': - # if timestamps.dtype.byteorder == '>': - # timestamps = timestamps.byteswap().newbyteorder() - # for signal in signals: - # if signal.samples.dtype.byteorder == '>': - # signal.samples = signal.samples.byteswap().newbyteorder() - if self.version >= "3.00": channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE elif self.version >= "2.10": @@ -2870,7 +2864,7 @@ def get( if data_type in v23c.INT_TYPES: - dtype_fmt = get_fmt_v3(data_type, bits) + dtype_fmt = get_fmt_v3(data_type, bits, self.identification.byte_order) channel_dtype = dtype(dtype_fmt.split(")")[-1]) view = f'{channel_dtype.byteorder}u{vals.itemsize}' @@ -2897,7 +2891,7 @@ def get( ) else: if kind_ in "ui": - dtype_fmt = get_fmt_v3(data_type, bits) + dtype_fmt = get_fmt_v3(data_type, bits, self.identification.byte_order) channel_dtype = dtype(dtype_fmt.split(")")[-1]) vals = vals.view(channel_dtype) @@ -2911,7 +2905,7 @@ def get( vals = array(vals, dtype=bool) else: data_type = channel.data_type - channel_dtype = array([], dtype=get_fmt_v3(data_type, bits)) + channel_dtype = array([], dtype=get_fmt_v3(data_type, bits, self.identification.byte_order)) if vals.dtype != channel_dtype.dtype: try: vals = vals.astype(channel_dtype.dtype) diff --git a/asammdf/blocks/utils.py b/asammdf/blocks/utils.py index d70fda7c5..94d6909f0 100644 --- a/asammdf/blocks/utils.py +++ b/asammdf/blocks/utils.py @@ -9,6 +9,7 @@ import re import subprocess from io import BytesIO +import sys from collections import namedtuple from random import randint @@ -263,7 +264,7 @@ def sanitize_xml(text): return re.sub(_xmlns_pattern, "", text) -def get_fmt_v3(data_type, size): +def get_fmt_v3(data_type, size, byte_order=v3c.BYTE_ORDER_INTEL): """convert mdf versions 2 and 3 channel data type to numpy dtype format string @@ -279,14 +280,14 @@ def get_fmt_v3(data_type, size): numpy compatible data type format string """ - if data_type in {v3c.DATA_TYPE_STRING, v3c.DATA_TYPE_BYTEARRAY}: + if data_type in (v3c.DATA_TYPE_STRING, v3c.DATA_TYPE_BYTEARRAY): size = size // 8 if data_type == v3c.DATA_TYPE_STRING: fmt = f"S{size}" else: fmt = f"({size},)u1" else: - if size > 64 and data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED): + if size > 64 and data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED, v3c.DATA_TYPE_UNSIGNED_MOTOROLA): fmt = f"({size // 8},)u1" else: if size <= 8: @@ -300,29 +301,42 @@ def get_fmt_v3(data_type, size): else: size = size // 8 - if data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED): + if data_type == v3c.DATA_TYPE_UNSIGNED_INTEL: fmt = f"u{size}" + elif data_type == v3c.DATA_TYPE_UNSIGNED_MOTOROLA: fmt = f">u{size}" - elif data_type in (v3c.DATA_TYPE_SIGNED_INTEL, v3c.DATA_TYPE_SIGNED): + elif data_type == v3c.DATA_TYPE_SIGNED_INTEL: fmt = f"i{size}" + elif data_type == v3c.DATA_TYPE_SIGNED_MOTOROLA: fmt = f">i{size}" - elif data_type in { - v3c.DATA_TYPE_FLOAT, - v3c.DATA_TYPE_DOUBLE, - v3c.DATA_TYPE_FLOAT_INTEL, - v3c.DATA_TYPE_DOUBLE_INTEL, - }: + elif data_type in (v3c.DATA_TYPE_FLOAT_INTEL, v3c.DATA_TYPE_DOUBLE_INTEL): fmt = f"f{size}" + elif data_type in (v3c.DATA_TYPE_FLOAT, v3c.DATA_TYPE_DOUBLE): + if byte_order == v3c.BYTE_ORDER_INTEL: + fmt = f"f{size}" + return fmt @@ -430,25 +444,29 @@ def fmt_to_datatype_v3(fmt, shape, array=False): integer data type as defined by ASAM MDF and bit size """ + byteorder = fmt.byteorder + if byteorder == '=': + byteorder = '<' if sys.byteorder == 'little' else '>' size = fmt.itemsize * 8 + kind = fmt.kind - if not array and shape[1:] and fmt.itemsize == 1 and fmt.kind == "u": + if not array and shape[1:] and fmt.itemsize == 1 and kind == "u": data_type = v3c.DATA_TYPE_BYTEARRAY for dim in shape[1:]: size *= dim else: - if fmt.kind == "u": - if fmt.byteorder in "=<|": - data_type = v3c.DATA_TYPE_UNSIGNED + if kind == "u": + if byteorder in "<|": + data_type = v3c.DATA_TYPE_UNSIGNED_INTEL else: data_type = v3c.DATA_TYPE_UNSIGNED_MOTOROLA - elif fmt.kind == "i": - if fmt.byteorder in "=<|": - data_type = v3c.DATA_TYPE_SIGNED + elif kind == "i": + if byteorder in "<|": + data_type = v3c.DATA_TYPE_SIGNED_INTEL else: data_type = v3c.DATA_TYPE_SIGNED_MOTOROLA - elif fmt.kind == "f": - if fmt.byteorder in "=<": + elif kind == "f": + if byteorder in "<": if size == 32: data_type = v3c.DATA_TYPE_FLOAT else: @@ -458,10 +476,10 @@ def fmt_to_datatype_v3(fmt, shape, array=False): data_type = v3c.DATA_TYPE_FLOAT_MOTOROLA else: data_type = v3c.DATA_TYPE_DOUBLE_MOTOROLA - elif fmt.kind in "SV": + elif kind in "SV": data_type = v3c.DATA_TYPE_STRING - elif fmt.kind == "b": - data_type = v3c.DATA_TYPE_UNSIGNED + elif kind == "b": + data_type = v3c.DATA_TYPE_UNSIGNED_INTEL size = 1 else: message = f"Unknown type: dtype={fmt}, shape={shape}" @@ -521,28 +539,30 @@ def fmt_to_datatype_v4(fmt, shape, array=False): integer data type as defined by ASAM MDF and bit size """ + byteorder = fmt.byteorder + if byteorder == '=': + byteorder = '<' if sys.byteorder == 'little' else '>' size = fmt.itemsize * 8 + kind = fmt.kind - if not array and shape[1:] and fmt.itemsize == 1 and fmt.kind == "u": + if not array and shape[1:] and fmt.itemsize == 1 and kind == "u": data_type = v4c.DATA_TYPE_BYTEARRAY for dim in shape[1:]: size *= dim else: - kind = fmt.kind - byteorder = fmt.byteorder if kind == "u": - if byteorder in "=<|": + if byteorder in "<|": data_type = v4c.DATA_TYPE_UNSIGNED_INTEL else: data_type = v4c.DATA_TYPE_UNSIGNED_MOTOROLA elif kind == "i": - if byteorder in "=<|": + if byteorder in "<|": data_type = v4c.DATA_TYPE_SIGNED_INTEL else: data_type = v4c.DATA_TYPE_SIGNED_MOTOROLA elif kind == "f": - if byteorder in "=<": + if byteorder in "<": data_type = v4c.DATA_TYPE_REAL_INTEL else: data_type = v4c.DATA_TYPE_REAL_MOTOROLA @@ -552,7 +572,7 @@ def fmt_to_datatype_v4(fmt, shape, array=False): data_type = v4c.DATA_TYPE_UNSIGNED_INTEL size = 1 elif kind == "c": - if byteorder in "=<": + if byteorder in "<": data_type = v4c.DATA_TYPE_COMPLEX_INTEL else: data_type = v4c.DATA_TYPE_COMPLEX_MOTOROLA diff --git a/asammdf/blocks/v2_v3_blocks.py b/asammdf/blocks/v2_v3_blocks.py index e18871a44..40f5d8267 100644 --- a/asammdf/blocks/v2_v3_blocks.py +++ b/asammdf/blocks/v2_v3_blocks.py @@ -7,6 +7,7 @@ from getpass import getuser from struct import pack, unpack, unpack_from from textwrap import wrap +import sys import numpy as np from numexpr import evaluate @@ -2514,7 +2515,7 @@ def __init__(self, **kwargs): self.program_identification = "amdf{}".format( __version__.replace(".", "") ).encode("latin-1") - self.byte_order = v23c.BYTE_ORDER_INTEL + self.byte_order = v23c.BYTE_ORDER_INTEL if sys.byteorder == 'little' else v23c.BYTE_ORDER_MOTOROLA self.float_format = 0 self.mdf_version = int(version.replace(".", "")) self.code_page = 0 diff --git a/asammdf/mdf.py b/asammdf/mdf.py index 9cdf1e451..2219ac9e0 100644 --- a/asammdf/mdf.py +++ b/asammdf/mdf.py @@ -613,7 +613,9 @@ def cut( ] else: for sig in signals: - sig.samples = sig.samples.astype(sig.samples.dtype.newbyteorder('=')) + native = sig.samples.dtype.newbyteorder('=') + if sig.samples.dtype != native: + sig.samples = sig.samples.astype(native) if time_from_zero: master = master - delta diff --git a/asammdf/version.py b/asammdf/version.py index 9f591815b..7d43e9afa 100644 --- a/asammdf/version.py +++ b/asammdf/version.py @@ -1,4 +1,4 @@ # -*- coding: utf-8 -*- """ asammdf version module """ -__version__ = "5.19.6" +__version__ = "5.19.7"