Skip to content

Commit

Permalink
fix default byte order
Browse files Browse the repository at this point in the history
  • Loading branch information
@danielhrisca
danielhrisca committed Mar 19, 2020
1 parent df40545 commit 40e6ef9
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 56 deletions.
40 changes: 17 additions & 23 deletions asammdf/blocks/mdf_v3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,8 @@ def _prepare_record(self, group):
"""

byte_order = self.identification.byte_order

parents, dtypes = group.parents, group.types
if parents is None:
grp = group
Expand Down Expand Up @@ -457,7 +459,7 @@ def _prepare_record(self, group):
if data_type == v23c.DATA_TYPE_STRING:
next_byte_aligned_position = parent_start_offset + size
if next_byte_aligned_position <= record_size:
dtype_pair = (name, get_fmt_v3(data_type, size))
dtype_pair = (name, get_fmt_v3(data_type, size, byte_order))
types.append(dtype_pair)
parents[original_index] = name, bit_offset

Expand All @@ -467,7 +469,7 @@ def _prepare_record(self, group):
elif data_type == v23c.DATA_TYPE_BYTEARRAY:
next_byte_aligned_position = parent_start_offset + size
if next_byte_aligned_position <= record_size:
dtype_pair = (name, get_fmt_v3(data_type, size))
dtype_pair = (name, get_fmt_v3(data_type, size, byte_order))
types.append(dtype_pair)
parents[original_index] = name, bit_offset
else:
Expand All @@ -479,7 +481,10 @@ def _prepare_record(self, group):
byte_size += 1
bit_size = byte_size * 8

if data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA):
if (
data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA)
or data_type in (v23c.DATA_TYPE_SIGNED, v23c.DATA_TYPE_UNSIGNED) and byte_order == v23c.BYTE_ORDER_MOTOROLA
):

if size > 32:
next_byte_aligned_position = parent_start_offset + 64
Expand All @@ -504,7 +509,7 @@ def _prepare_record(self, group):
next_byte_aligned_position = parent_start_offset + 8

if next_byte_aligned_position <= record_size:
dtype_pair = (name, get_fmt_v3(data_type, size))
dtype_pair = (name, get_fmt_v3(data_type, size, byte_order))
types.append(dtype_pair)
parents[original_index] = name, bit_offset
else:
Expand All @@ -522,7 +527,10 @@ def _prepare_record(self, group):

max_overlapping = next_byte_aligned_position - byte_start_offset
if max_overlapping >= bit_size:
if data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA):
if (
data_type in(v23c.DATA_TYPE_SIGNED_MOTOROLA, v23c.DATA_TYPE_UNSIGNED_MOTOROLA)
or data_type in (v23c.DATA_TYPE_SIGNED, v23c.DATA_TYPE_UNSIGNED) and byte_order == v23c.BYTE_ORDER_MOTOROLA
):
parents[original_index] = (
current_parent,
bit_offset + max_overlapping - bit_size,
Expand Down Expand Up @@ -640,7 +648,7 @@ def _get_not_byte_aligned_data(self, data, group, ch_nr):
if data_type in v23c.SIGNED_INT:
return as_non_byte_sized_signed_int(vals, bit_count)
elif data_type in v23c.FLOATS:
return vals.view(get_fmt_v3(data_type, bit_count))
return vals.view(get_fmt_v3(data_type, bit_count, self.identification.byte_order))
else:
return vals

Expand Down Expand Up @@ -1137,13 +1145,6 @@ def append(
else:
timestamps = array([])

if self.version < "3.10":
if timestamps.dtype.byteorder == ">":
timestamps = timestamps.byteswap().newbyteorder()
for signal in signals:
if signal.samples.dtype.byteorder == ">":
signal.samples = signal.samples.byteswap().newbyteorder()

if self.version >= "3.00":
channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE
elif self.version >= "2.10":
Expand Down Expand Up @@ -1985,13 +1986,6 @@ def _append_dataframe(self, df, source_info="", units=None):

timestamps = t

# if self.version < '3.10':
# if timestamps.dtype.byteorder == '>':
# timestamps = timestamps.byteswap().newbyteorder()
# for signal in signals:
# if signal.samples.dtype.byteorder == '>':
# signal.samples = signal.samples.byteswap().newbyteorder()

if self.version >= "3.00":
channel_size = v23c.CN_DISPLAYNAME_BLOCK_SIZE
elif self.version >= "2.10":
Expand Down Expand Up @@ -2870,7 +2864,7 @@ def get(

if data_type in v23c.INT_TYPES:

dtype_fmt = get_fmt_v3(data_type, bits)
dtype_fmt = get_fmt_v3(data_type, bits, self.identification.byte_order)
channel_dtype = dtype(dtype_fmt.split(")")[-1])

view = f'{channel_dtype.byteorder}u{vals.itemsize}'
Expand All @@ -2897,7 +2891,7 @@ def get(
)
else:
if kind_ in "ui":
dtype_fmt = get_fmt_v3(data_type, bits)
dtype_fmt = get_fmt_v3(data_type, bits, self.identification.byte_order)
channel_dtype = dtype(dtype_fmt.split(")")[-1])
vals = vals.view(channel_dtype)

Expand All @@ -2911,7 +2905,7 @@ def get(
vals = array(vals, dtype=bool)
else:
data_type = channel.data_type
channel_dtype = array([], dtype=get_fmt_v3(data_type, bits))
channel_dtype = array([], dtype=get_fmt_v3(data_type, bits, self.identification.byte_order))
if vals.dtype != channel_dtype.dtype:
try:
vals = vals.astype(channel_dtype.dtype)
Expand Down
80 changes: 50 additions & 30 deletions asammdf/blocks/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import re
import subprocess
from io import BytesIO
import sys

from collections import namedtuple
from random import randint
Expand Down Expand Up @@ -263,7 +264,7 @@ def sanitize_xml(text):
return re.sub(_xmlns_pattern, "", text)


def get_fmt_v3(data_type, size):
def get_fmt_v3(data_type, size, byte_order=v3c.BYTE_ORDER_INTEL):
"""convert mdf versions 2 and 3 channel data type to numpy dtype format
string
Expand All @@ -279,14 +280,14 @@ def get_fmt_v3(data_type, size):
numpy compatible data type format string
"""
if data_type in {v3c.DATA_TYPE_STRING, v3c.DATA_TYPE_BYTEARRAY}:
if data_type in (v3c.DATA_TYPE_STRING, v3c.DATA_TYPE_BYTEARRAY):
size = size // 8
if data_type == v3c.DATA_TYPE_STRING:
fmt = f"S{size}"
else:
fmt = f"({size},)u1"
else:
if size > 64 and data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED):
if size > 64 and data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED, v3c.DATA_TYPE_UNSIGNED_MOTOROLA):
fmt = f"({size // 8},)u1"
else:
if size <= 8:
Expand All @@ -300,29 +301,42 @@ def get_fmt_v3(data_type, size):
else:
size = size // 8

if data_type in (v3c.DATA_TYPE_UNSIGNED_INTEL, v3c.DATA_TYPE_UNSIGNED):
if data_type == v3c.DATA_TYPE_UNSIGNED_INTEL:
fmt = f"<u{size}"

elif data_type == v3c.DATA_TYPE_UNSIGNED:
if byte_order == v3c.BYTE_ORDER_INTEL:
fmt = f"<u{size}"
else:
fmt = f">u{size}"

elif data_type == v3c.DATA_TYPE_UNSIGNED_MOTOROLA:
fmt = f">u{size}"

elif data_type in (v3c.DATA_TYPE_SIGNED_INTEL, v3c.DATA_TYPE_SIGNED):
elif data_type == v3c.DATA_TYPE_SIGNED_INTEL:
fmt = f"<i{size}"

elif data_type == v3c.DATA_TYPE_SIGNED:
if byte_order == v3c.BYTE_ORDER_INTEL:
fmt = f"<i{size}"
else:
fmt = f">i{size}"

elif data_type == v3c.DATA_TYPE_SIGNED_MOTOROLA:
fmt = f">i{size}"

elif data_type in {
v3c.DATA_TYPE_FLOAT,
v3c.DATA_TYPE_DOUBLE,
v3c.DATA_TYPE_FLOAT_INTEL,
v3c.DATA_TYPE_DOUBLE_INTEL,
}:
elif data_type in (v3c.DATA_TYPE_FLOAT_INTEL, v3c.DATA_TYPE_DOUBLE_INTEL):
fmt = f"<f{size}"

elif data_type in (v3c.DATA_TYPE_FLOAT_MOTOROLA, v3c.DATA_TYPE_DOUBLE_MOTOROLA):
fmt = f">f{size}"

elif data_type in (v3c.DATA_TYPE_FLOAT, v3c.DATA_TYPE_DOUBLE):
if byte_order == v3c.BYTE_ORDER_INTEL:
fmt = f"<f{size}"
else:
fmt = f">f{size}"

return fmt


Expand Down Expand Up @@ -430,25 +444,29 @@ def fmt_to_datatype_v3(fmt, shape, array=False):
integer data type as defined by ASAM MDF and bit size
"""
byteorder = fmt.byteorder
if byteorder == '=':
byteorder = '<' if sys.byteorder == 'little' else '>'
size = fmt.itemsize * 8
kind = fmt.kind

if not array and shape[1:] and fmt.itemsize == 1 and fmt.kind == "u":
if not array and shape[1:] and fmt.itemsize == 1 and kind == "u":
data_type = v3c.DATA_TYPE_BYTEARRAY
for dim in shape[1:]:
size *= dim
else:
if fmt.kind == "u":
if fmt.byteorder in "=<|":
data_type = v3c.DATA_TYPE_UNSIGNED
if kind == "u":
if byteorder in "<|":
data_type = v3c.DATA_TYPE_UNSIGNED_INTEL
else:
data_type = v3c.DATA_TYPE_UNSIGNED_MOTOROLA
elif fmt.kind == "i":
if fmt.byteorder in "=<|":
data_type = v3c.DATA_TYPE_SIGNED
elif kind == "i":
if byteorder in "<|":
data_type = v3c.DATA_TYPE_SIGNED_INTEL
else:
data_type = v3c.DATA_TYPE_SIGNED_MOTOROLA
elif fmt.kind == "f":
if fmt.byteorder in "=<":
elif kind == "f":
if byteorder in "<":
if size == 32:
data_type = v3c.DATA_TYPE_FLOAT
else:
Expand All @@ -458,10 +476,10 @@ def fmt_to_datatype_v3(fmt, shape, array=False):
data_type = v3c.DATA_TYPE_FLOAT_MOTOROLA
else:
data_type = v3c.DATA_TYPE_DOUBLE_MOTOROLA
elif fmt.kind in "SV":
elif kind in "SV":
data_type = v3c.DATA_TYPE_STRING
elif fmt.kind == "b":
data_type = v3c.DATA_TYPE_UNSIGNED
elif kind == "b":
data_type = v3c.DATA_TYPE_UNSIGNED_INTEL
size = 1
else:
message = f"Unknown type: dtype={fmt}, shape={shape}"
Expand Down Expand Up @@ -521,28 +539,30 @@ def fmt_to_datatype_v4(fmt, shape, array=False):
integer data type as defined by ASAM MDF and bit size
"""
byteorder = fmt.byteorder
if byteorder == '=':
byteorder = '<' if sys.byteorder == 'little' else '>'
size = fmt.itemsize * 8
kind = fmt.kind

if not array and shape[1:] and fmt.itemsize == 1 and fmt.kind == "u":
if not array and shape[1:] and fmt.itemsize == 1 and kind == "u":
data_type = v4c.DATA_TYPE_BYTEARRAY
for dim in shape[1:]:
size *= dim

else:
kind = fmt.kind
byteorder = fmt.byteorder
if kind == "u":
if byteorder in "=<|":
if byteorder in "<|":
data_type = v4c.DATA_TYPE_UNSIGNED_INTEL
else:
data_type = v4c.DATA_TYPE_UNSIGNED_MOTOROLA
elif kind == "i":
if byteorder in "=<|":
if byteorder in "<|":
data_type = v4c.DATA_TYPE_SIGNED_INTEL
else:
data_type = v4c.DATA_TYPE_SIGNED_MOTOROLA
elif kind == "f":
if byteorder in "=<":
if byteorder in "<":
data_type = v4c.DATA_TYPE_REAL_INTEL
else:
data_type = v4c.DATA_TYPE_REAL_MOTOROLA
Expand All @@ -552,7 +572,7 @@ def fmt_to_datatype_v4(fmt, shape, array=False):
data_type = v4c.DATA_TYPE_UNSIGNED_INTEL
size = 1
elif kind == "c":
if byteorder in "=<":
if byteorder in "<":
data_type = v4c.DATA_TYPE_COMPLEX_INTEL
else:
data_type = v4c.DATA_TYPE_COMPLEX_MOTOROLA
Expand Down
3 changes: 2 additions & 1 deletion asammdf/blocks/v2_v3_blocks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from getpass import getuser
from struct import pack, unpack, unpack_from
from textwrap import wrap
import sys

import numpy as np
from numexpr import evaluate
Expand Down Expand Up @@ -2514,7 +2515,7 @@ def __init__(self, **kwargs):
self.program_identification = "amdf{}".format(
__version__.replace(".", "")
).encode("latin-1")
self.byte_order = v23c.BYTE_ORDER_INTEL
self.byte_order = v23c.BYTE_ORDER_INTEL if sys.byteorder == 'little' else v23c.BYTE_ORDER_MOTOROLA
self.float_format = 0
self.mdf_version = int(version.replace(".", ""))
self.code_page = 0
Expand Down
4 changes: 3 additions & 1 deletion asammdf/mdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,9 @@ def cut(
]
else:
for sig in signals:
sig.samples = sig.samples.astype(sig.samples.dtype.newbyteorder('='))
native = sig.samples.dtype.newbyteorder('=')
if sig.samples.dtype != native:
sig.samples = sig.samples.astype(native)

if time_from_zero:
master = master - delta
Expand Down
2 changes: 1 addition & 1 deletion asammdf/version.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# -*- coding: utf-8 -*-
""" asammdf version module """

__version__ = "5.19.6"
__version__ = "5.19.7"

0 comments on commit 40e6ef9

Please sign in to comment.