From f4762075256a08431763fe6f271b776bab1f9e35 Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Fri, 6 Mar 2020 19:56:35 +0200 Subject: [PATCH 1/3] Implement https://github.com/UAVCAN/specification/issues/75 --- pydsdl/__init__.py | 2 +- pydsdl/_serializable/_array.py | 33 ++------ pydsdl/_serializable/_composite.py | 120 +++++++++++++++-------------- pydsdl/_test.py | 42 +++++----- 4 files changed, 90 insertions(+), 107 deletions(-) diff --git a/pydsdl/__init__.py b/pydsdl/__init__.py index 9b0c812..347aafe 100644 --- a/pydsdl/__init__.py +++ b/pydsdl/__init__.py @@ -32,7 +32,7 @@ file=_sys.stderr) _sys.exit(1) -__version__ = '1.1.0' +__version__ = '1.2.0' __version_info__ = tuple(map(int, __version__.split('.'))) __license__ = 'MIT' __author__ = 'UAVCAN Development Team' diff --git a/pydsdl/_serializable/_array.py b/pydsdl/_serializable/_array.py index 750246d..384e183 100644 --- a/pydsdl/_serializable/_array.py +++ b/pydsdl/_serializable/_array.py @@ -4,6 +4,7 @@ # import abc +import math import typing from .._bit_length_set import BitLengthSet from ._serializable import SerializableType, TypeParameterError @@ -123,7 +124,8 @@ def __init__(self, capacity: int): super(VariableLengthArrayType, self).__init__(element_type, capacity) # Construct once to allow reference equality checks - self._length_field_type = UnsignedIntegerType(self.capacity.bit_length(), PrimitiveType.CastMode.TRUNCATED) + length_field_length = 2 ** math.ceil(math.log2(max(8, self.capacity.bit_length()))) + self._length_field_type = UnsignedIntegerType(length_field_length, PrimitiveType.CastMode.TRUNCATED) @property def string_like(self) -> bool: @@ -172,8 +174,8 @@ def _unittest_variable_array() -> None: assert not VariableLengthArrayType(si64, 1).string_like # Mind the length prefix! - assert VariableLengthArrayType(tu8, 3).bit_length_set == {2, 10, 18, 26} - assert VariableLengthArrayType(tu8, 1).bit_length_set == {1, 9} + assert VariableLengthArrayType(tu8, 3).bit_length_set == {8, 16, 24, 32} + assert VariableLengthArrayType(tu8, 1).bit_length_set == {8, 16} assert max(VariableLengthArrayType(tu8, 255).bit_length_set) == 2048 assert VariableLengthArrayType(tu8, 200).capacity == 200 @@ -186,27 +188,8 @@ def _unittest_variable_array() -> None: 'VariableLengthArrayType(element_type=SignedIntegerType(bit_length=64, cast_mode=), ' \ 'capacity=128)' - # The following was computed manually; it is easy to validate: - # we have zero, one, or two elements of 8 bits each; plus 2 bit wide tag; therefore: - # {2 + 0, 2 + 8, 2 + 16} small = VariableLengthArrayType(tu8, 2) - assert small.bit_length_set == {2, 10, 18} - - # This one gets a little tricky, so pull out a piece of paper an a pencil. - # So the nested type, as defined above, has the following set: {2, 10, 18}. - # We can have up to two elements of that type, so what we get can be expressed graphically as follows: - # A B | + - # ---------+------ - # 2 2 | 4 - # 10 2 | 12 - # 18 2 | 20 - # 2 10 | 12 - # 10 10 | 20 - # 18 10 | 28 - # 2 18 | 20 - # 10 18 | 28 - # 18 18 | 36 - # - # If we were to remove duplicates, we end up with: {4, 12, 20, 28, 36} + assert small.bit_length_set == {8, 16, 24} + outer = FixedLengthArrayType(small, 2) - assert outer.bit_length_set == {4, 12, 20, 28, 36} + assert outer.bit_length_set == {16, 24, 32, 40, 48} diff --git a/pydsdl/_serializable/_composite.py b/pydsdl/_serializable/_composite.py index 4e1ff5c..18a2c73 100644 --- a/pydsdl/_serializable/_composite.py +++ b/pydsdl/_serializable/_composite.py @@ -4,6 +4,7 @@ # import abc +import math import typing import itertools from .. import _expression @@ -341,7 +342,8 @@ def __init__(self, # Construct once to allow reference equality checks assert (self.number_of_variants - 1) > 0 - tag_bit_length = (self.number_of_variants - 1).bit_length() + unaligned_tag_bit_length = (self.number_of_variants - 1).bit_length() + tag_bit_length = 2 ** math.ceil(math.log2(max(8, unaligned_tag_bit_length))) self._tag_field_type = UnsignedIntegerType(tag_bit_length, PrimitiveType.CastMode.TRUNCATED) @property @@ -594,7 +596,7 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType: assert try_union_fields([ outer, SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), - ]).bit_length_set == {5, 13, 17, 21, 29, 37} + ]).bit_length_set == {17, 25, 33, 41, 49} def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureType: atr = [] @@ -618,9 +620,9 @@ def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureTy assert try_struct_fields([ outer, SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), - ]).bit_length_set == {4 + 16, 12 + 16, 20 + 16, 28 + 16, 36 + 16} + ]).bit_length_set == {16 + 16, 24 + 16, 32 + 16, 40 + 16, 48 + 16} - assert try_struct_fields([outer]).bit_length_set == {4, 12, 20, 28, 36} + assert try_struct_fields([outer]).bit_length_set == {16, 24, 32, 40, 48} def _unittest_field_iterators() -> None: @@ -662,21 +664,21 @@ def validate_iterator(t: CompositeType, ('b', {10}), ('c', {11}), ('d', { - 11 + 2 + 32 * 0, - 11 + 2 + 32 * 1, - 11 + 2 + 32 * 2, + 11 + 8 + 32 * 0, + 11 + 8 + 32 * 1, + 11 + 8 + 32 * 2, }), ('', { - 11 + 2 + 32 * 0 + 32 * 7, - 11 + 2 + 32 * 1 + 32 * 7, - 11 + 2 + 32 * 2 + 32 * 7, + 11 + 8 + 32 * 0 + 32 * 7, + 11 + 8 + 32 * 1 + 32 * 7, + 11 + 8 + 32 * 2 + 32 * 7, }), ]) a_bls_options = [ - 11 + 2 + 32 * 0 + 32 * 7 + 3, - 11 + 2 + 32 * 1 + 32 * 7 + 3, - 11 + 2 + 32 * 2 + 32 * 7 + 3, + 11 + 8 + 32 * 0 + 32 * 7 + 3, + 11 + 8 + 32 * 1 + 32 * 7 + 3, + 11 + 8 + 32 * 2 + 32 * 7 + 3, ] assert a.bit_length_set == BitLengthSet(a_bls_options) @@ -686,20 +688,20 @@ def validate_iterator(t: CompositeType, ('b', {1 + 10, 16 + 10}), ('c', {1 + 11, 16 + 11}), ('d', { - 1 + 11 + 2 + 32 * 0, - 1 + 11 + 2 + 32 * 1, - 1 + 11 + 2 + 32 * 2, - 16 + 11 + 2 + 32 * 0, - 16 + 11 + 2 + 32 * 1, - 16 + 11 + 2 + 32 * 2, + 1 + 11 + 8 + 32 * 0, + 1 + 11 + 8 + 32 * 1, + 1 + 11 + 8 + 32 * 2, + 16 + 11 + 8 + 32 * 0, + 16 + 11 + 8 + 32 * 1, + 16 + 11 + 8 + 32 * 2, }), ('', { - 1 + 11 + 2 + 32 * 0 + 32 * 7, - 1 + 11 + 2 + 32 * 1 + 32 * 7, - 1 + 11 + 2 + 32 * 2 + 32 * 7, - 16 + 11 + 2 + 32 * 0 + 32 * 7, - 16 + 11 + 2 + 32 * 1 + 32 * 7, - 16 + 11 + 2 + 32 * 2 + 32 * 7, + 1 + 11 + 8 + 32 * 0 + 32 * 7, + 1 + 11 + 8 + 32 * 1 + 32 * 7, + 1 + 11 + 8 + 32 * 2 + 32 * 7, + 16 + 11 + 8 + 32 * 0 + 32 * 7, + 16 + 11 + 8 + 32 * 1 + 32 * 7, + 16 + 11 + 8 + 32 * 2 + 32 * 7, }), ], BitLengthSet({1, 16})) @@ -718,35 +720,35 @@ def validate_iterator(t: CompositeType, }), ('x', { # The lone "+2" is for the variable-length array's implicit length field # First length option of z - a_bls_options[0] + 2 + a_bls_options[0] * 0, # suka - a_bls_options[0] + 2 + a_bls_options[1] * 0, - a_bls_options[0] + 2 + a_bls_options[2] * 0, - a_bls_options[0] + 2 + a_bls_options[0] * 1, - a_bls_options[0] + 2 + a_bls_options[1] * 1, - a_bls_options[0] + 2 + a_bls_options[2] * 1, - a_bls_options[0] + 2 + a_bls_options[0] * 2, - a_bls_options[0] + 2 + a_bls_options[1] * 2, - a_bls_options[0] + 2 + a_bls_options[2] * 2, + a_bls_options[0] + 8 + a_bls_options[0] * 0, # suka + a_bls_options[0] + 8 + a_bls_options[1] * 0, + a_bls_options[0] + 8 + a_bls_options[2] * 0, + a_bls_options[0] + 8 + a_bls_options[0] * 1, + a_bls_options[0] + 8 + a_bls_options[1] * 1, + a_bls_options[0] + 8 + a_bls_options[2] * 1, + a_bls_options[0] + 8 + a_bls_options[0] * 2, + a_bls_options[0] + 8 + a_bls_options[1] * 2, + a_bls_options[0] + 8 + a_bls_options[2] * 2, # Second length option of z - a_bls_options[1] + 2 + a_bls_options[0] * 0, - a_bls_options[1] + 2 + a_bls_options[1] * 0, - a_bls_options[1] + 2 + a_bls_options[2] * 0, - a_bls_options[1] + 2 + a_bls_options[0] * 1, - a_bls_options[1] + 2 + a_bls_options[1] * 1, - a_bls_options[1] + 2 + a_bls_options[2] * 1, - a_bls_options[1] + 2 + a_bls_options[0] * 2, - a_bls_options[1] + 2 + a_bls_options[1] * 2, - a_bls_options[1] + 2 + a_bls_options[2] * 2, + a_bls_options[1] + 8 + a_bls_options[0] * 0, + a_bls_options[1] + 8 + a_bls_options[1] * 0, + a_bls_options[1] + 8 + a_bls_options[2] * 0, + a_bls_options[1] + 8 + a_bls_options[0] * 1, + a_bls_options[1] + 8 + a_bls_options[1] * 1, + a_bls_options[1] + 8 + a_bls_options[2] * 1, + a_bls_options[1] + 8 + a_bls_options[0] * 2, + a_bls_options[1] + 8 + a_bls_options[1] * 2, + a_bls_options[1] + 8 + a_bls_options[2] * 2, # Third length option of z - a_bls_options[2] + 2 + a_bls_options[0] * 0, - a_bls_options[2] + 2 + a_bls_options[1] * 0, - a_bls_options[2] + 2 + a_bls_options[2] * 0, - a_bls_options[2] + 2 + a_bls_options[0] * 1, - a_bls_options[2] + 2 + a_bls_options[1] * 1, - a_bls_options[2] + 2 + a_bls_options[2] * 1, - a_bls_options[2] + 2 + a_bls_options[0] * 2, - a_bls_options[2] + 2 + a_bls_options[1] * 2, - a_bls_options[2] + 2 + a_bls_options[2] * 2, + a_bls_options[2] + 8 + a_bls_options[0] * 0, + a_bls_options[2] + 8 + a_bls_options[1] * 0, + a_bls_options[2] + 8 + a_bls_options[2] * 0, + a_bls_options[2] + 8 + a_bls_options[0] * 1, + a_bls_options[2] + 8 + a_bls_options[1] * 1, + a_bls_options[2] + 8 + a_bls_options[2] * 1, + a_bls_options[2] + 8 + a_bls_options[0] * 2, + a_bls_options[2] + 8 + a_bls_options[1] * 2, + a_bls_options[2] + 8 + a_bls_options[2] * 2, }), ]) @@ -756,7 +758,7 @@ def validate_iterator(t: CompositeType, b_offset.increment(f.data_type.bit_length_set) print('b_offset:', b_offset) assert b_offset == b.bit_length_set - assert b_offset.is_aligned_at_byte() + assert not b_offset.is_aligned_at_byte() assert not b_offset.is_aligned_at(32) c = make_type(UnionType, [ @@ -765,18 +767,18 @@ def validate_iterator(t: CompositeType, ]) validate_iterator(c, [ - ('foo', {1}), # The offset is the same because it's a union - ('bar', {1}), + ('foo', {8}), # The offset is the same because it's a union + ('bar', {8}), ]) validate_iterator(c, [ - ('foo', {8 + 1}), - ('bar', {8 + 1}), + ('foo', {8 + 8}), + ('bar', {8 + 8}), ], BitLengthSet(8)) validate_iterator(c, [ - ('foo', {0 + 1, 4 + 1, 8 + 1}), - ('bar', {0 + 1, 4 + 1, 8 + 1}), + ('foo', {0 + 8, 4 + 8, 8 + 8}), + ('bar', {0 + 8, 4 + 8, 8 + 8}), ], BitLengthSet({0, 4, 8})) with raises(TypeError, match='.*request or response.*'): diff --git a/pydsdl/_test.py b/pydsdl/_test.py index ac6eac5..76d4ae1 100644 --- a/pydsdl/_test.py +++ b/pydsdl/_test.py @@ -101,8 +101,8 @@ def _unittest_simple() -> None: assert p.fixed_port_id == 29000 assert p.deprecated assert p.version == (1, 2) - assert min(p.bit_length_set) == 14 - assert max(p.bit_length_set) == 14 + 64 * 32 + assert min(p.bit_length_set) == 16 + assert max(p.bit_length_set) == 16 + 64 * 32 assert len(p.attributes) == 3 assert len(p.fields) == 2 assert str(p.fields[0].data_type) == 'saturated int8' @@ -206,8 +206,8 @@ def _unittest_simple() -> None: assert res.deprecated assert not res.has_fixed_port_id assert res.version == (0, 1) - assert min(res.bit_length_set) == 14 - assert max(res.bit_length_set) == 14 + 64 * 32 + assert min(res.bit_length_set) == 16 + assert max(res.bit_length_set) == 16 + 64 * 32 t = res.fields[0].data_type assert isinstance(t, _serializable.StructureType) @@ -522,27 +522,26 @@ def _unittest_assert() -> None: @assert _offset_ == {0} @assert _offset_.min == _offset_.max Array.1.0[2] bar - @assert _offset_ == {4, 12, 20, 28, 36} - @assert _offset_.min == 4 - @assert _offset_.max == 36 - @assert _offset_ % 4 == {0} - @assert _offset_ % 8 == {4} - @assert _offset_ % 10 == {4, 2, 0, 8, 6} - @assert _offset_ * 2 == {8, 24, 40, 56, 72} - @assert 2 * _offset_ == {8, 24, 40, 56, 72} - @assert _offset_ / 4 == {1, 3, 5, 7, 9} - @assert _offset_ - 4 == {0, 8, 16, 24, 32} - @assert _offset_ + 4 == {8, 16, 24, 32, 40} + @assert _offset_ == {16, 24, 32, 40, 48} + @assert _offset_.min == 16 + @assert _offset_.max == 48 + @assert _offset_ % 8 == {0} + @assert _offset_ % 10 == {6, 4, 2, 0, 8} + @assert _offset_ * 2 == {32, 48, 64, 80, 96} + @assert 2 * _offset_ == {32, 48, 64, 80, 96} + @assert _offset_ / 4 == {4, 6, 8, 10, 12} + @assert _offset_ - 4 == {12, 20, 28, 36, 44} + @assert _offset_ + 4 == {20, 28, 36, 44, 52} uint64 big - @assert _offset_ - 64 == {4, 12, 20, 28, 36} - @assert _offset_.min == 68 - @assert _offset_.max == 100 # 36 + 64 - @assert _offset_.max <= 100 - @assert _offset_.max < 101 + @assert _offset_ - 64 == {16, 24, 32, 40, 48} + @assert _offset_.min == 80 + @assert _offset_.max == 112 + @assert _offset_.max <= 112 + @assert _offset_.max < 113 @assert _offset_ == _offset_ @assert truncated uint64._bit_length_ == {64} @assert uint64._bit_length_ == {64} - @assert Array.1.0._bit_length_.max == 2 + 8 + 8 + @assert Array.1.0._bit_length_.max == 8 + 8 + 8 ''')), [ _define('ns/Array.1.0.uavcan', 'uint8[<=2] foo') @@ -689,7 +688,6 @@ def _define(rel_path: str, text: str) -> None: _define( 'zubax/29001.Message.1.0.uavcan', dedent(""" - void6 zubax.First.1.0[<=2] a @assert _offset_.min == 8 @assert _offset_.max == 4104 From 638a65ef87c8fc0d3873effd034d55e1e5eb989a Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Sat, 7 Mar 2020 22:27:09 +0200 Subject: [PATCH 2/3] Update BitLengthSet to use the new union tag size computation logic --- pydsdl/_bit_length_set.py | 4 +++- pydsdl/_serializable/_composite.py | 4 ++-- pydsdl/_test.py | 12 ++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pydsdl/_bit_length_set.py b/pydsdl/_bit_length_set.py index ab79e0a..a7af816 100644 --- a/pydsdl/_bit_length_set.py +++ b/pydsdl/_bit_length_set.py @@ -3,6 +3,7 @@ # This software is distributed under the terms of the MIT License. # +import math import typing import itertools @@ -212,7 +213,8 @@ def for_tagged_union(member_bit_length_sets: typing.Iterable[typing.Union[typing for s in ms: out.unite_with(s) # Add the union tag: - out.increment((len(ms) - 1).bit_length()) + tag_bit_length = 2 ** math.ceil(math.log2(max(8, (len(ms) - 1).bit_length()))) + out.increment(tag_bit_length) return out diff --git a/pydsdl/_serializable/_composite.py b/pydsdl/_serializable/_composite.py index 18a2c73..a65d009 100644 --- a/pydsdl/_serializable/_composite.py +++ b/pydsdl/_serializable/_composite.py @@ -585,7 +585,7 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType: assert try_union_fields([ UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED), SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), - ]).bit_length_set == {17} + ]).bit_length_set == {24} # The reference values for the following test are explained in the array tests above tu8 = UnsignedIntegerType(8, cast_mode=PrimitiveType.CastMode.TRUNCATED) @@ -596,7 +596,7 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType: assert try_union_fields([ outer, SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), - ]).bit_length_set == {17, 25, 33, 41, 49} + ]).bit_length_set == {24, 32, 40, 48, 56} def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureType: atr = [] diff --git a/pydsdl/_test.py b/pydsdl/_test.py index 76d4ae1..5281fb3 100644 --- a/pydsdl/_test.py +++ b/pydsdl/_test.py @@ -183,7 +183,7 @@ def _unittest_simple() -> None: assert req.deprecated assert not req.has_fixed_port_id assert req.version == (0, 1) - assert req.bit_length_set == 2 # Remember this is a union + assert req.bit_length_set == 8 # Remember this is a union assert [x.name for x in req.fields] == ['new_empty_implicit', 'new_empty_explicit', 'old_empty'] t = req.fields[0].data_type @@ -255,8 +255,8 @@ def _unittest_simple() -> None: assert len(p.constants) == 1 assert p.constants[0].name == 'PI' assert str(p.constants[0].data_type) == 'truncated float16' - assert min(p.bit_length_set) == 2 - assert max(p.bit_length_set) == 2 + 8 + 255 + assert min(p.bit_length_set) == 8 + assert max(p.bit_length_set) == 8 + 8 + 255 assert len(p.fields) == 3 assert str(p.fields[0]) == 'saturated uint8 a' assert str(p.fields[1]) == 'vendor.nested.Empty.255.255[5] b' @@ -601,7 +601,7 @@ def _unittest_assert() -> None: @union float32 a uint64 b - @assert _offset_ == {33, 65} + @assert _offset_ == {40, 72} ''')), [] ) @@ -616,7 +616,7 @@ def _unittest_assert() -> None: uint8 B = 1 uint64 b uint8 C = 2 - @assert _offset_ == {33, 65} + @assert _offset_ == {40, 72} uint8 D = 3 ''')), [] @@ -631,7 +631,7 @@ def _unittest_assert() -> None: @assert _offset_.min == 33 float32 a uint64 b - @assert _offset_ == {33, 65} + @assert _offset_ == {40, 72} ''')), [] ) From db246cfa00c578afe103d0f0453691b395f9a1ab Mon Sep 17 00:00:00 2001 From: Pavel Kirienko Date: Mon, 9 Mar 2020 20:12:09 +0200 Subject: [PATCH 3/3] Add additional tests for implicit prefixes longer than 8 bit --- pydsdl/_serializable/_array.py | 5 +++++ pydsdl/_serializable/_composite.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pydsdl/_serializable/_array.py b/pydsdl/_serializable/_array.py index 384e183..998a2d5 100644 --- a/pydsdl/_serializable/_array.py +++ b/pydsdl/_serializable/_array.py @@ -193,3 +193,8 @@ def _unittest_variable_array() -> None: outer = FixedLengthArrayType(small, 2) assert outer.bit_length_set == {16, 24, 32, 40, 48} + + assert VariableLengthArrayType(tu8, 100).length_field_type.bit_length == 8 + assert VariableLengthArrayType(tu8, 10000).length_field_type.bit_length == 16 + assert VariableLengthArrayType(tu8, 1000000).length_field_type.bit_length == 32 + assert VariableLengthArrayType(tu8, 10000000000).length_field_type.bit_length == 64 diff --git a/pydsdl/_serializable/_composite.py b/pydsdl/_serializable/_composite.py index a65d009..6cc0a9a 100644 --- a/pydsdl/_serializable/_composite.py +++ b/pydsdl/_serializable/_composite.py @@ -587,6 +587,20 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType: SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), ]).bit_length_set == {24} + assert try_union_fields( + [ + UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED), + SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), + ] * 1000 + ).bit_length_set == {16 + 16} + + assert try_union_fields( + [ + UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED), + SignedIntegerType(16, PrimitiveType.CastMode.SATURATED), + ] * 1000000 + ).bit_length_set == {32 + 16} + # The reference values for the following test are explained in the array tests above tu8 = UnsignedIntegerType(8, cast_mode=PrimitiveType.CastMode.TRUNCATED) small = VariableLengthArrayType(tu8, 2)