Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement implicit field size alignment #33

Merged
merged 3 commits into from
Mar 9, 2020
Merged

Implement implicit field size alignment #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f476207
Implement https://github.com/UAVCAN/specification/issues/75
pavel-kirienko Mar 6, 2020
638a65e
Update BitLengthSet to use the new union tag size computation logic
pavel-kirienko Mar 7, 2020
db246cf
Add additional tests for implicit prefixes longer than 8 bit
pavel-kirienko Mar 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pydsdl/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
file=_sys.stderr)
_sys.exit(1)

__version__ = '1.1.0'
__version__ = '1.2.0'
__version_info__ = tuple(map(int, __version__.split('.')))
__license__ = 'MIT'
__author__ = 'UAVCAN Development Team'
Expand Down
4 changes: 3 additions & 1 deletion pydsdl/_bit_length_set.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# This software is distributed under the terms of the MIT License.
#

import math
import typing
import itertools

Expand Down Expand Up @@ -212,7 +213,8 @@ def for_tagged_union(member_bit_length_sets: typing.Iterable[typing.Union[typing
for s in ms:
out.unite_with(s)
# Add the union tag:
out.increment((len(ms) - 1).bit_length())
tag_bit_length = 2 ** math.ceil(math.log2(max(8, (len(ms) - 1).bit_length())))
out.increment(tag_bit_length)

return out

Expand Down
38 changes: 13 additions & 25 deletions pydsdl/_serializable/_array.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#

import abc
import math
import typing
from .._bit_length_set import BitLengthSet
from ._serializable import SerializableType, TypeParameterError
Expand Down Expand Up @@ -123,7 +124,8 @@ def __init__(self,
capacity: int):
super(VariableLengthArrayType, self).__init__(element_type, capacity)
# Construct once to allow reference equality checks
self._length_field_type = UnsignedIntegerType(self.capacity.bit_length(), PrimitiveType.CastMode.TRUNCATED)
length_field_length = 2 ** math.ceil(math.log2(max(8, self.capacity.bit_length())))
self._length_field_type = UnsignedIntegerType(length_field_length, PrimitiveType.CastMode.TRUNCATED)

@property
def string_like(self) -> bool:
Expand Down Expand Up @@ -172,8 +174,8 @@ def _unittest_variable_array() -> None:
assert not VariableLengthArrayType(si64, 1).string_like

# Mind the length prefix!
assert VariableLengthArrayType(tu8, 3).bit_length_set == {2, 10, 18, 26}
assert VariableLengthArrayType(tu8, 1).bit_length_set == {1, 9}
assert VariableLengthArrayType(tu8, 3).bit_length_set == {8, 16, 24, 32}
assert VariableLengthArrayType(tu8, 1).bit_length_set == {8, 16}
assert max(VariableLengthArrayType(tu8, 255).bit_length_set) == 2048

assert VariableLengthArrayType(tu8, 200).capacity == 200
Expand All @@ -186,27 +188,13 @@ def _unittest_variable_array() -> None:
'VariableLengthArrayType(element_type=SignedIntegerType(bit_length=64, cast_mode=<CastMode.SATURATED: 0>), ' \
'capacity=128)'

# The following was computed manually; it is easy to validate:
# we have zero, one, or two elements of 8 bits each; plus 2 bit wide tag; therefore:
# {2 + 0, 2 + 8, 2 + 16}
small = VariableLengthArrayType(tu8, 2)
assert small.bit_length_set == {2, 10, 18}

# This one gets a little tricky, so pull out a piece of paper an a pencil.
# So the nested type, as defined above, has the following set: {2, 10, 18}.
# We can have up to two elements of that type, so what we get can be expressed graphically as follows:
# A B | +
# ---------+------
# 2 2 | 4
# 10 2 | 12
# 18 2 | 20
# 2 10 | 12
# 10 10 | 20
# 18 10 | 28
# 2 18 | 20
# 10 18 | 28
# 18 18 | 36
#
# If we were to remove duplicates, we end up with: {4, 12, 20, 28, 36}
assert small.bit_length_set == {8, 16, 24}

outer = FixedLengthArrayType(small, 2)
assert outer.bit_length_set == {4, 12, 20, 28, 36}
assert outer.bit_length_set == {16, 24, 32, 40, 48}

assert VariableLengthArrayType(tu8, 100).length_field_type.bit_length == 8
assert VariableLengthArrayType(tu8, 10000).length_field_type.bit_length == 16
assert VariableLengthArrayType(tu8, 1000000).length_field_type.bit_length == 32
assert VariableLengthArrayType(tu8, 10000000000).length_field_type.bit_length == 64
136 changes: 76 additions & 60 deletions pydsdl/_serializable/_composite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#

import abc
import math
import typing
import itertools
from .. import _expression
Expand Down Expand Up @@ -341,7 +342,8 @@ def __init__(self,

# Construct once to allow reference equality checks
assert (self.number_of_variants - 1) > 0
tag_bit_length = (self.number_of_variants - 1).bit_length()
unaligned_tag_bit_length = (self.number_of_variants - 1).bit_length()
tag_bit_length = 2 ** math.ceil(math.log2(max(8, unaligned_tag_bit_length)))
self._tag_field_type = UnsignedIntegerType(tag_bit_length, PrimitiveType.CastMode.TRUNCATED)

@property
Expand Down Expand Up @@ -583,7 +585,21 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType:
assert try_union_fields([
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {17}
]).bit_length_set == {24}

assert try_union_fields(
[
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
] * 1000
).bit_length_set == {16 + 16}

assert try_union_fields(
[
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
] * 1000000
).bit_length_set == {32 + 16}

# The reference values for the following test are explained in the array tests above
tu8 = UnsignedIntegerType(8, cast_mode=PrimitiveType.CastMode.TRUNCATED)
Expand All @@ -594,7 +610,7 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType:
assert try_union_fields([
outer,
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {5, 13, 17, 21, 29, 37}
]).bit_length_set == {24, 32, 40, 48, 56}

def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureType:
atr = []
Expand All @@ -618,9 +634,9 @@ def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureTy
assert try_struct_fields([
outer,
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {4 + 16, 12 + 16, 20 + 16, 28 + 16, 36 + 16}
]).bit_length_set == {16 + 16, 24 + 16, 32 + 16, 40 + 16, 48 + 16}

assert try_struct_fields([outer]).bit_length_set == {4, 12, 20, 28, 36}
assert try_struct_fields([outer]).bit_length_set == {16, 24, 32, 40, 48}


def _unittest_field_iterators() -> None:
Expand Down Expand Up @@ -662,21 +678,21 @@ def validate_iterator(t: CompositeType,
('b', {10}),
('c', {11}),
('d', {
11 + 2 + 32 * 0,
11 + 2 + 32 * 1,
11 + 2 + 32 * 2,
11 + 8 + 32 * 0,
11 + 8 + 32 * 1,
11 + 8 + 32 * 2,
}),
('', {
11 + 2 + 32 * 0 + 32 * 7,
11 + 2 + 32 * 1 + 32 * 7,
11 + 2 + 32 * 2 + 32 * 7,
11 + 8 + 32 * 0 + 32 * 7,
11 + 8 + 32 * 1 + 32 * 7,
11 + 8 + 32 * 2 + 32 * 7,
}),
])

a_bls_options = [
11 + 2 + 32 * 0 + 32 * 7 + 3,
11 + 2 + 32 * 1 + 32 * 7 + 3,
11 + 2 + 32 * 2 + 32 * 7 + 3,
11 + 8 + 32 * 0 + 32 * 7 + 3,
11 + 8 + 32 * 1 + 32 * 7 + 3,
11 + 8 + 32 * 2 + 32 * 7 + 3,
]
assert a.bit_length_set == BitLengthSet(a_bls_options)

Expand All @@ -686,20 +702,20 @@ def validate_iterator(t: CompositeType,
('b', {1 + 10, 16 + 10}),
('c', {1 + 11, 16 + 11}),
('d', {
1 + 11 + 2 + 32 * 0,
1 + 11 + 2 + 32 * 1,
1 + 11 + 2 + 32 * 2,
16 + 11 + 2 + 32 * 0,
16 + 11 + 2 + 32 * 1,
16 + 11 + 2 + 32 * 2,
1 + 11 + 8 + 32 * 0,
1 + 11 + 8 + 32 * 1,
1 + 11 + 8 + 32 * 2,
16 + 11 + 8 + 32 * 0,
16 + 11 + 8 + 32 * 1,
16 + 11 + 8 + 32 * 2,
}),
('', {
1 + 11 + 2 + 32 * 0 + 32 * 7,
1 + 11 + 2 + 32 * 1 + 32 * 7,
1 + 11 + 2 + 32 * 2 + 32 * 7,
16 + 11 + 2 + 32 * 0 + 32 * 7,
16 + 11 + 2 + 32 * 1 + 32 * 7,
16 + 11 + 2 + 32 * 2 + 32 * 7,
1 + 11 + 8 + 32 * 0 + 32 * 7,
1 + 11 + 8 + 32 * 1 + 32 * 7,
1 + 11 + 8 + 32 * 2 + 32 * 7,
16 + 11 + 8 + 32 * 0 + 32 * 7,
16 + 11 + 8 + 32 * 1 + 32 * 7,
16 + 11 + 8 + 32 * 2 + 32 * 7,
}),
], BitLengthSet({1, 16}))

Expand All @@ -718,35 +734,35 @@ def validate_iterator(t: CompositeType,
}),
('x', { # The lone "+2" is for the variable-length array's implicit length field
# First length option of z
a_bls_options[0] + 2 + a_bls_options[0] * 0, # suka
a_bls_options[0] + 2 + a_bls_options[1] * 0,
a_bls_options[0] + 2 + a_bls_options[2] * 0,
a_bls_options[0] + 2 + a_bls_options[0] * 1,
a_bls_options[0] + 2 + a_bls_options[1] * 1,
a_bls_options[0] + 2 + a_bls_options[2] * 1,
a_bls_options[0] + 2 + a_bls_options[0] * 2,
a_bls_options[0] + 2 + a_bls_options[1] * 2,
a_bls_options[0] + 2 + a_bls_options[2] * 2,
a_bls_options[0] + 8 + a_bls_options[0] * 0, # suka
a_bls_options[0] + 8 + a_bls_options[1] * 0,
a_bls_options[0] + 8 + a_bls_options[2] * 0,
a_bls_options[0] + 8 + a_bls_options[0] * 1,
a_bls_options[0] + 8 + a_bls_options[1] * 1,
a_bls_options[0] + 8 + a_bls_options[2] * 1,
a_bls_options[0] + 8 + a_bls_options[0] * 2,
a_bls_options[0] + 8 + a_bls_options[1] * 2,
a_bls_options[0] + 8 + a_bls_options[2] * 2,
# Second length option of z
a_bls_options[1] + 2 + a_bls_options[0] * 0,
a_bls_options[1] + 2 + a_bls_options[1] * 0,
a_bls_options[1] + 2 + a_bls_options[2] * 0,
a_bls_options[1] + 2 + a_bls_options[0] * 1,
a_bls_options[1] + 2 + a_bls_options[1] * 1,
a_bls_options[1] + 2 + a_bls_options[2] * 1,
a_bls_options[1] + 2 + a_bls_options[0] * 2,
a_bls_options[1] + 2 + a_bls_options[1] * 2,
a_bls_options[1] + 2 + a_bls_options[2] * 2,
a_bls_options[1] + 8 + a_bls_options[0] * 0,
a_bls_options[1] + 8 + a_bls_options[1] * 0,
a_bls_options[1] + 8 + a_bls_options[2] * 0,
a_bls_options[1] + 8 + a_bls_options[0] * 1,
a_bls_options[1] + 8 + a_bls_options[1] * 1,
a_bls_options[1] + 8 + a_bls_options[2] * 1,
a_bls_options[1] + 8 + a_bls_options[0] * 2,
a_bls_options[1] + 8 + a_bls_options[1] * 2,
a_bls_options[1] + 8 + a_bls_options[2] * 2,
# Third length option of z
a_bls_options[2] + 2 + a_bls_options[0] * 0,
a_bls_options[2] + 2 + a_bls_options[1] * 0,
a_bls_options[2] + 2 + a_bls_options[2] * 0,
a_bls_options[2] + 2 + a_bls_options[0] * 1,
a_bls_options[2] + 2 + a_bls_options[1] * 1,
a_bls_options[2] + 2 + a_bls_options[2] * 1,
a_bls_options[2] + 2 + a_bls_options[0] * 2,
a_bls_options[2] + 2 + a_bls_options[1] * 2,
a_bls_options[2] + 2 + a_bls_options[2] * 2,
a_bls_options[2] + 8 + a_bls_options[0] * 0,
a_bls_options[2] + 8 + a_bls_options[1] * 0,
a_bls_options[2] + 8 + a_bls_options[2] * 0,
a_bls_options[2] + 8 + a_bls_options[0] * 1,
a_bls_options[2] + 8 + a_bls_options[1] * 1,
a_bls_options[2] + 8 + a_bls_options[2] * 1,
a_bls_options[2] + 8 + a_bls_options[0] * 2,
a_bls_options[2] + 8 + a_bls_options[1] * 2,
a_bls_options[2] + 8 + a_bls_options[2] * 2,
}),
])

Expand All @@ -756,7 +772,7 @@ def validate_iterator(t: CompositeType,
b_offset.increment(f.data_type.bit_length_set)
print('b_offset:', b_offset)
assert b_offset == b.bit_length_set
assert b_offset.is_aligned_at_byte()
assert not b_offset.is_aligned_at_byte()
assert not b_offset.is_aligned_at(32)

c = make_type(UnionType, [
Expand All @@ -765,18 +781,18 @@ def validate_iterator(t: CompositeType,
])

validate_iterator(c, [
('foo', {1}), # The offset is the same because it's a union
('bar', {1}),
('foo', {8}), # The offset is the same because it's a union
('bar', {8}),
])

validate_iterator(c, [
('foo', {8 + 1}),
('bar', {8 + 1}),
('foo', {8 + 8}),
('bar', {8 + 8}),
], BitLengthSet(8))

validate_iterator(c, [
('foo', {0 + 1, 4 + 1, 8 + 1}),
('bar', {0 + 1, 4 + 1, 8 + 1}),
('foo', {0 + 8, 4 + 8, 8 + 8}),
('bar', {0 + 8, 4 + 8, 8 + 8}),
], BitLengthSet({0, 4, 8}))

with raises(TypeError, match='.*request or response.*'):
Expand Down
Loading