Skip to content

Commit

Permalink
Merge pull request #33 from UAVCAN/implicit-field-size-alignment
Browse files Browse the repository at this point in the history 
Implement implicit field size alignment
  • Loading branch information
@pavel-kirienko
pavel-kirienko authored Mar 9, 2020
2 parents 81c32e6 + db246cf commit 897762d
Show file tree
Hide file tree
Showing 5 changed files with 119 additions and 115 deletions.
2 changes: 1 addition & 1 deletion pydsdl/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
file=_sys.stderr)
_sys.exit(1)

__version__ = '1.1.0'
__version__ = '1.2.0'
__version_info__ = tuple(map(int, __version__.split('.')))
__license__ = 'MIT'
__author__ = 'UAVCAN Development Team'
Expand Down
4 changes: 3 additions & 1 deletion pydsdl/_bit_length_set.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# This software is distributed under the terms of the MIT License.
#

import math
import typing
import itertools

Expand Down Expand Up @@ -212,7 +213,8 @@ def for_tagged_union(member_bit_length_sets: typing.Iterable[typing.Union[typing
for s in ms:
out.unite_with(s)
# Add the union tag:
out.increment((len(ms) - 1).bit_length())
tag_bit_length = 2 ** math.ceil(math.log2(max(8, (len(ms) - 1).bit_length())))
out.increment(tag_bit_length)

return out

Expand Down
38 changes: 13 additions & 25 deletions pydsdl/_serializable/_array.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#

import abc
import math
import typing
from .._bit_length_set import BitLengthSet
from ._serializable import SerializableType, TypeParameterError
Expand Down Expand Up @@ -123,7 +124,8 @@ def __init__(self,
capacity: int):
super(VariableLengthArrayType, self).__init__(element_type, capacity)
# Construct once to allow reference equality checks
self._length_field_type = UnsignedIntegerType(self.capacity.bit_length(), PrimitiveType.CastMode.TRUNCATED)
length_field_length = 2 ** math.ceil(math.log2(max(8, self.capacity.bit_length())))
self._length_field_type = UnsignedIntegerType(length_field_length, PrimitiveType.CastMode.TRUNCATED)

@property
def string_like(self) -> bool:
Expand Down Expand Up @@ -172,8 +174,8 @@ def _unittest_variable_array() -> None:
assert not VariableLengthArrayType(si64, 1).string_like

# Mind the length prefix!
assert VariableLengthArrayType(tu8, 3).bit_length_set == {2, 10, 18, 26}
assert VariableLengthArrayType(tu8, 1).bit_length_set == {1, 9}
assert VariableLengthArrayType(tu8, 3).bit_length_set == {8, 16, 24, 32}
assert VariableLengthArrayType(tu8, 1).bit_length_set == {8, 16}
assert max(VariableLengthArrayType(tu8, 255).bit_length_set) == 2048

assert VariableLengthArrayType(tu8, 200).capacity == 200
Expand All @@ -186,27 +188,13 @@ def _unittest_variable_array() -> None:
'VariableLengthArrayType(element_type=SignedIntegerType(bit_length=64, cast_mode=<CastMode.SATURATED: 0>), ' \
'capacity=128)'

# The following was computed manually; it is easy to validate:
# we have zero, one, or two elements of 8 bits each; plus 2 bit wide tag; therefore:
# {2 + 0, 2 + 8, 2 + 16}
small = VariableLengthArrayType(tu8, 2)
assert small.bit_length_set == {2, 10, 18}

# This one gets a little tricky, so pull out a piece of paper an a pencil.
# So the nested type, as defined above, has the following set: {2, 10, 18}.
# We can have up to two elements of that type, so what we get can be expressed graphically as follows:
# A B | +
# ---------+------
# 2 2 | 4
# 10 2 | 12
# 18 2 | 20
# 2 10 | 12
# 10 10 | 20
# 18 10 | 28
# 2 18 | 20
# 10 18 | 28
# 18 18 | 36
#
# If we were to remove duplicates, we end up with: {4, 12, 20, 28, 36}
assert small.bit_length_set == {8, 16, 24}

outer = FixedLengthArrayType(small, 2)
assert outer.bit_length_set == {4, 12, 20, 28, 36}
assert outer.bit_length_set == {16, 24, 32, 40, 48}

assert VariableLengthArrayType(tu8, 100).length_field_type.bit_length == 8
assert VariableLengthArrayType(tu8, 10000).length_field_type.bit_length == 16
assert VariableLengthArrayType(tu8, 1000000).length_field_type.bit_length == 32
assert VariableLengthArrayType(tu8, 10000000000).length_field_type.bit_length == 64
136 changes: 76 additions & 60 deletions pydsdl/_serializable/_composite.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#

import abc
import math
import typing
import itertools
from .. import _expression
Expand Down Expand Up @@ -341,7 +342,8 @@ def __init__(self,

# Construct once to allow reference equality checks
assert (self.number_of_variants - 1) > 0
tag_bit_length = (self.number_of_variants - 1).bit_length()
unaligned_tag_bit_length = (self.number_of_variants - 1).bit_length()
tag_bit_length = 2 ** math.ceil(math.log2(max(8, unaligned_tag_bit_length)))
self._tag_field_type = UnsignedIntegerType(tag_bit_length, PrimitiveType.CastMode.TRUNCATED)

@property
Expand Down Expand Up @@ -583,7 +585,21 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType:
assert try_union_fields([
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {17}
]).bit_length_set == {24}

assert try_union_fields(
[
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
] * 1000
).bit_length_set == {16 + 16}

assert try_union_fields(
[
UnsignedIntegerType(16, PrimitiveType.CastMode.TRUNCATED),
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
] * 1000000
).bit_length_set == {32 + 16}

# The reference values for the following test are explained in the array tests above
tu8 = UnsignedIntegerType(8, cast_mode=PrimitiveType.CastMode.TRUNCATED)
Expand All @@ -594,7 +610,7 @@ def try_union_fields(field_types: typing.List[SerializableType]) -> UnionType:
assert try_union_fields([
outer,
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {5, 13, 17, 21, 29, 37}
]).bit_length_set == {24, 32, 40, 48, 56}

def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureType:
atr = []
Expand All @@ -618,9 +634,9 @@ def try_struct_fields(field_types: typing.List[SerializableType]) -> StructureTy
assert try_struct_fields([
outer,
SignedIntegerType(16, PrimitiveType.CastMode.SATURATED),
]).bit_length_set == {4 + 16, 12 + 16, 20 + 16, 28 + 16, 36 + 16}
]).bit_length_set == {16 + 16, 24 + 16, 32 + 16, 40 + 16, 48 + 16}

assert try_struct_fields([outer]).bit_length_set == {4, 12, 20, 28, 36}
assert try_struct_fields([outer]).bit_length_set == {16, 24, 32, 40, 48}


def _unittest_field_iterators() -> None:
Expand Down Expand Up @@ -662,21 +678,21 @@ def validate_iterator(t: CompositeType,
('b', {10}),
('c', {11}),
('d', {
11 + 2 + 32 * 0,
11 + 2 + 32 * 1,
11 + 2 + 32 * 2,
11 + 8 + 32 * 0,
11 + 8 + 32 * 1,
11 + 8 + 32 * 2,
}),
('', {
11 + 2 + 32 * 0 + 32 * 7,
11 + 2 + 32 * 1 + 32 * 7,
11 + 2 + 32 * 2 + 32 * 7,
11 + 8 + 32 * 0 + 32 * 7,
11 + 8 + 32 * 1 + 32 * 7,
11 + 8 + 32 * 2 + 32 * 7,
}),
])

a_bls_options = [
11 + 2 + 32 * 0 + 32 * 7 + 3,
11 + 2 + 32 * 1 + 32 * 7 + 3,
11 + 2 + 32 * 2 + 32 * 7 + 3,
11 + 8 + 32 * 0 + 32 * 7 + 3,
11 + 8 + 32 * 1 + 32 * 7 + 3,
11 + 8 + 32 * 2 + 32 * 7 + 3,
]
assert a.bit_length_set == BitLengthSet(a_bls_options)

Expand All @@ -686,20 +702,20 @@ def validate_iterator(t: CompositeType,
('b', {1 + 10, 16 + 10}),
('c', {1 + 11, 16 + 11}),
('d', {
1 + 11 + 2 + 32 * 0,
1 + 11 + 2 + 32 * 1,
1 + 11 + 2 + 32 * 2,
16 + 11 + 2 + 32 * 0,
16 + 11 + 2 + 32 * 1,
16 + 11 + 2 + 32 * 2,
1 + 11 + 8 + 32 * 0,
1 + 11 + 8 + 32 * 1,
1 + 11 + 8 + 32 * 2,
16 + 11 + 8 + 32 * 0,
16 + 11 + 8 + 32 * 1,
16 + 11 + 8 + 32 * 2,
}),
('', {
1 + 11 + 2 + 32 * 0 + 32 * 7,
1 + 11 + 2 + 32 * 1 + 32 * 7,
1 + 11 + 2 + 32 * 2 + 32 * 7,
16 + 11 + 2 + 32 * 0 + 32 * 7,
16 + 11 + 2 + 32 * 1 + 32 * 7,
16 + 11 + 2 + 32 * 2 + 32 * 7,
1 + 11 + 8 + 32 * 0 + 32 * 7,
1 + 11 + 8 + 32 * 1 + 32 * 7,
1 + 11 + 8 + 32 * 2 + 32 * 7,
16 + 11 + 8 + 32 * 0 + 32 * 7,
16 + 11 + 8 + 32 * 1 + 32 * 7,
16 + 11 + 8 + 32 * 2 + 32 * 7,
}),
], BitLengthSet({1, 16}))

Expand All @@ -718,35 +734,35 @@ def validate_iterator(t: CompositeType,
}),
('x', { # The lone "+2" is for the variable-length array's implicit length field
# First length option of z
a_bls_options[0] + 2 + a_bls_options[0] * 0, # suka
a_bls_options[0] + 2 + a_bls_options[1] * 0,
a_bls_options[0] + 2 + a_bls_options[2] * 0,
a_bls_options[0] + 2 + a_bls_options[0] * 1,
a_bls_options[0] + 2 + a_bls_options[1] * 1,
a_bls_options[0] + 2 + a_bls_options[2] * 1,
a_bls_options[0] + 2 + a_bls_options[0] * 2,
a_bls_options[0] + 2 + a_bls_options[1] * 2,
a_bls_options[0] + 2 + a_bls_options[2] * 2,
a_bls_options[0] + 8 + a_bls_options[0] * 0, # suka
a_bls_options[0] + 8 + a_bls_options[1] * 0,
a_bls_options[0] + 8 + a_bls_options[2] * 0,
a_bls_options[0] + 8 + a_bls_options[0] * 1,
a_bls_options[0] + 8 + a_bls_options[1] * 1,
a_bls_options[0] + 8 + a_bls_options[2] * 1,
a_bls_options[0] + 8 + a_bls_options[0] * 2,
a_bls_options[0] + 8 + a_bls_options[1] * 2,
a_bls_options[0] + 8 + a_bls_options[2] * 2,
# Second length option of z
a_bls_options[1] + 2 + a_bls_options[0] * 0,
a_bls_options[1] + 2 + a_bls_options[1] * 0,
a_bls_options[1] + 2 + a_bls_options[2] * 0,
a_bls_options[1] + 2 + a_bls_options[0] * 1,
a_bls_options[1] + 2 + a_bls_options[1] * 1,
a_bls_options[1] + 2 + a_bls_options[2] * 1,
a_bls_options[1] + 2 + a_bls_options[0] * 2,
a_bls_options[1] + 2 + a_bls_options[1] * 2,
a_bls_options[1] + 2 + a_bls_options[2] * 2,
a_bls_options[1] + 8 + a_bls_options[0] * 0,
a_bls_options[1] + 8 + a_bls_options[1] * 0,
a_bls_options[1] + 8 + a_bls_options[2] * 0,
a_bls_options[1] + 8 + a_bls_options[0] * 1,
a_bls_options[1] + 8 + a_bls_options[1] * 1,
a_bls_options[1] + 8 + a_bls_options[2] * 1,
a_bls_options[1] + 8 + a_bls_options[0] * 2,
a_bls_options[1] + 8 + a_bls_options[1] * 2,
a_bls_options[1] + 8 + a_bls_options[2] * 2,
# Third length option of z
a_bls_options[2] + 2 + a_bls_options[0] * 0,
a_bls_options[2] + 2 + a_bls_options[1] * 0,
a_bls_options[2] + 2 + a_bls_options[2] * 0,
a_bls_options[2] + 2 + a_bls_options[0] * 1,
a_bls_options[2] + 2 + a_bls_options[1] * 1,
a_bls_options[2] + 2 + a_bls_options[2] * 1,
a_bls_options[2] + 2 + a_bls_options[0] * 2,
a_bls_options[2] + 2 + a_bls_options[1] * 2,
a_bls_options[2] + 2 + a_bls_options[2] * 2,
a_bls_options[2] + 8 + a_bls_options[0] * 0,
a_bls_options[2] + 8 + a_bls_options[1] * 0,
a_bls_options[2] + 8 + a_bls_options[2] * 0,
a_bls_options[2] + 8 + a_bls_options[0] * 1,
a_bls_options[2] + 8 + a_bls_options[1] * 1,
a_bls_options[2] + 8 + a_bls_options[2] * 1,
a_bls_options[2] + 8 + a_bls_options[0] * 2,
a_bls_options[2] + 8 + a_bls_options[1] * 2,
a_bls_options[2] + 8 + a_bls_options[2] * 2,
}),
])

Expand All @@ -756,7 +772,7 @@ def validate_iterator(t: CompositeType,
b_offset.increment(f.data_type.bit_length_set)
print('b_offset:', b_offset)
assert b_offset == b.bit_length_set
assert b_offset.is_aligned_at_byte()
assert not b_offset.is_aligned_at_byte()
assert not b_offset.is_aligned_at(32)

c = make_type(UnionType, [
Expand All @@ -765,18 +781,18 @@ def validate_iterator(t: CompositeType,
])

validate_iterator(c, [
('foo', {1}), # The offset is the same because it's a union
('bar', {1}),
('foo', {8}), # The offset is the same because it's a union
('bar', {8}),
])

validate_iterator(c, [
('foo', {8 + 1}),
('bar', {8 + 1}),
('foo', {8 + 8}),
('bar', {8 + 8}),
], BitLengthSet(8))

validate_iterator(c, [
('foo', {0 + 1, 4 + 1, 8 + 1}),
('bar', {0 + 1, 4 + 1, 8 + 1}),
('foo', {0 + 8, 4 + 8, 8 + 8}),
('bar', {0 + 8, 4 + 8, 8 + 8}),
], BitLengthSet({0, 4, 8}))

with raises(TypeError, match='.*request or response.*'):
Expand Down
Loading

0 comments on commit 897762d

Please sign in to comment.