From 88e2d9cd5ea301848554cb51729a77b0c2d46807 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 10:41:12 -0500 Subject: [PATCH 1/8] add level of indirection to _encode and _decode This allows the entry point to be inlined at the caller site, paving the way for various optimizations. _encode now calls the non-inlined __encode function in the .c file which does the preliminary work then calls ___encode to encode the actual type, if compound (or e.g. canardEncodeScalar). --- templates/msg.c.em | 8 ++++---- templates/msg.h.em | 38 +++++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/templates/msg.c.em b/templates/msg.c.em index 1b7d9cd..aa3f57e 100644 --- a/templates/msg.c.em +++ b/templates/msg.c.em @@ -11,14 +11,14 @@ #include #endif -uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer +uint32_t _@(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer #if CANARD_ENABLE_TAO_OPTION , bool tao #endif ) { uint32_t bit_ofs = 0; memset(buffer, 0, @(msg_define_name.upper())_MAX_SIZE); - _@(msg_underscored_name)_encode(buffer, &bit_ofs, msg, + __@(msg_underscored_name)_encode(buffer, &bit_ofs, msg, #if CANARD_ENABLE_TAO_OPTION tao #else @@ -31,14 +31,14 @@ uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer /* return true if the decode is invalid */ -bool @(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg) { +bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg) { #if CANARD_ENABLE_TAO_OPTION if (transfer->tao && (transfer->payload_len > @(msg_define_name.upper())_MAX_SIZE)) { return true; /* invalid payload length */ } #endif uint32_t bit_ofs = 0; - if (_@(msg_underscored_name)_decode(transfer, &bit_ofs, msg, + if (__@(msg_underscored_name)_decode(transfer, &bit_ofs, msg, #if CANARD_ENABLE_TAO_OPTION transfer->tao #else diff --git a/templates/msg.h.em b/templates/msg.h.em index 8ec0b2d..6d4b204 100644 --- a/templates/msg.h.em +++ b/templates/msg.h.em @@ -69,18 +69,34 @@ extern "C" { #endif -uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer +uint32_t _@(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer #if CANARD_ENABLE_TAO_OPTION , bool tao #endif ); -bool @(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg); +bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg); + +static inline uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer +#if CANARD_ENABLE_TAO_OPTION + , bool tao +#endif +) { + return _@(msg_underscored_name)_encode(msg, buffer +#if CANARD_ENABLE_TAO_OPTION + , tao +#endif + ); +} + +static inline bool @(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg) { + return _@(msg_underscored_name)_decode(transfer, msg); +} #if defined(CANARD_DSDLC_INTERNAL) @{indent = 0}@{ind = ' '*indent}@ -static inline void _@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao); -static inline bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao); -void _@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao) { +static inline void __@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao); +static inline bool __@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao); +void __@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao) { @{indent += 1}@{ind = ' '*indent}@ @(ind)(void)buffer; @(ind)(void)bit_ofs; @@ -101,7 +117,7 @@ void _@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c @{indent += 1}@{ind = ' '*indent}@ @[ end if]@ @[ if field.type.category == field.type.CATEGORY_COMPOUND]@ -@(ind)_@(underscored_name(field.type))_encode(buffer, bit_ofs, &msg->@(field.name), @('tao' if (field == msg_fields[-1] or msg_union) else 'false')); +@(ind)__@(underscored_name(field.type))_encode(buffer, bit_ofs, &msg->@(field.name), @('tao' if (field == msg_fields[-1] or msg_union) else 'false')); @[ elif field.type.category == field.type.CATEGORY_PRIMITIVE]@ @[ if field.type.kind == field.type.KIND_FLOAT and field.type.bitlen == 16]@ @(ind){ @@ -144,7 +160,7 @@ void _@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c @[ end if]@ @(ind)*bit_ofs += @(field.type.value_type.bitlen); @[ elif field.type.value_type.category == field.type.value_type.CATEGORY_COMPOUND]@ -@(ind)_@(underscored_name(field.type.value_type))_encode(buffer, bit_ofs, &msg->@(field_get_data(field))[i], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@); +@(ind)__@(underscored_name(field.type.value_type))_encode(buffer, bit_ofs, &msg->@(field_get_data(field))[i], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@); @[ end if]@ @{indent -= 1}@{ind = ' '*indent}@ @(ind)} @@ -167,7 +183,7 @@ void _@(msg_underscored_name)_encode(uint8_t* buffer, uint32_t* bit_ofs, @(msg_c /* decode @(msg_underscored_name), return true on failure, false on success */ -bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao) { +bool __@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* bit_ofs, @(msg_c_type)* msg, bool tao) { @{indent += 1}@{ind = ' '*indent}@ @(ind)(void)transfer; @(ind)(void)bit_ofs; @@ -194,7 +210,7 @@ bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* @{indent += 1}@{ind = ' '*indent}@ @[ end if]@ @[ if field.type.category == field.type.CATEGORY_COMPOUND]@ -@(ind)if (_@(underscored_name(field.type))_decode(transfer, bit_ofs, &msg->@(field.name), @('tao' if (field == msg_fields[-1] or msg_union) else 'false'))) {return true;} +@(ind)if (__@(underscored_name(field.type))_decode(transfer, bit_ofs, &msg->@(field.name), @('tao' if (field == msg_fields[-1] or msg_union) else 'false'))) {return true;} @[ elif field.type.category == field.type.CATEGORY_PRIMITIVE]@ @[ if field.type.kind == field.type.KIND_FLOAT and field.type.bitlen == 16]@ @(ind){ @@ -235,7 +251,7 @@ bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* @(ind)uint32_t max_bits = (transfer->payload_len*8)-7; // TAO elements must be >= 8 bits @(ind)while (max_bits > *bit_ofs) { @{indent += 1}@{ind = ' '*indent}@ -@(ind)if (!max_len-- || _@(underscored_name(field.type.value_type))_decode(transfer, bit_ofs, &msg->@(field_get_data(field))[msg->@(field.name).len], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@)) {return true;} +@(ind)if (!max_len-- || __@(underscored_name(field.type.value_type))_decode(transfer, bit_ofs, &msg->@(field_get_data(field))[msg->@(field.name).len], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@)) {return true;} @(ind)msg->@(field.name).len++; @{indent -= 1}@{ind = ' '*indent}@ @(ind)} @@ -269,7 +285,7 @@ bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, uint32_t* @[ end if]@ @(ind)*bit_ofs += @(field.type.value_type.bitlen); @[ elif field.type.value_type.category == field.type.value_type.CATEGORY_COMPOUND]@ -@(ind)if (_@(underscored_name(field.type.value_type))_decode(transfer, bit_ofs, &msg->@(field_get_data(field))[i], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@)) {return true;} +@(ind)if (__@(underscored_name(field.type.value_type))_decode(transfer, bit_ofs, &msg->@(field_get_data(field))[i], @[if field == msg_fields[-1] and field.type.value_type.get_min_bitlen() < 8]tao && i==msg->@(field.name).len@[else]false@[end if]@)) {return true;} @[ end if]@ @{indent -= 1}@{ind = ' '*indent}@ @(ind)} From eb2ae9b8a57fade916d9079eb32131ea8a4a7a72 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 10:59:47 -0500 Subject: [PATCH 2/8] inline coding of 0-length messages Allows the processing of 0-length messages (which is true of some important ones in the DSDL) to be optimized away. --- templates/msg.h.em | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/templates/msg.h.em b/templates/msg.h.em index 6d4b204..31412c8 100644 --- a/templates/msg.h.em +++ b/templates/msg.h.em @@ -81,15 +81,32 @@ static inline uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_ , bool tao #endif ) { +@[if msg_max_bitlen == 0] + (void)msg; + (void)buffer; +#if CANARD_ENABLE_TAO_OPTION + (void)tao; +#endif + + return 0; // 0-length message encodes to 0 bytes +@[else] return _@(msg_underscored_name)_encode(msg, buffer #if CANARD_ENABLE_TAO_OPTION , tao #endif ); +@[end if] } static inline bool @(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg) { +@[if msg_max_bitlen == 0] + (void)msg; + + // all transports accurately convey a payload length of 0 bytes so any payload is an error + return transfer->payload_len != 0; +@[else] return _@(msg_underscored_name)_decode(transfer, msg); +@[end if] } #if defined(CANARD_DSDLC_INTERNAL) From 042e1d71d9b7c39b23f586c7b5e78ca0a45f0a64 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 14:28:05 -0500 Subject: [PATCH 3/8] introduce skeleton for table coding system If the message structure is compatible, the compiler generates a table which describes the structure. If enabled, an interpreter in libcanard is then used to follow the table description and encode or decode the message. This occupies substantially fewer bytes per message as the table is much smaller than the C function previously used. This translates to a large reduction in program memory usage, especially if many different message types need to be coded. The call to the interpreter (along with a pointer to the message and the associated table) is inlined whereever e.g. _encode is called. The existing function to encode/decode is still generated in case table coding is not enabled in libcanard or the structure is not compatible. This commit sets up all the hooks to generate the tables and call the interpreter functions. However, no tables are generated as the generation function is a simple stub that always fails. --- dronecan_dsdlc_helpers.py | 3 +++ templates/msg.c.em | 12 ++++++++++++ templates/msg.h.em | 21 +++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index 010bd68..534b36c 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -270,3 +270,6 @@ def mkdir_p(path): pass else: raise + +def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fields): + return None diff --git a/templates/msg.c.em b/templates/msg.c.em index aa3f57e..34efa3e 100644 --- a/templates/msg.c.em +++ b/templates/msg.c.em @@ -11,6 +11,18 @@ #include #endif +@{coding_table = build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fields)} +@[if coding_table is not None] +#if CANARD_ENABLE_TABLE_ENCODING || CANARD_ENABLE_TABLE_DECODING +const CanardCodingTable _@(msg_underscored_name)_coding_table = { + .entries_max = @(len(coding_table))-1, + .entries = { +@("\n".join(" "+e for e in coding_table)) + }, +}; +#endif +@[end if] + uint32_t _@(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer #if CANARD_ENABLE_TAO_OPTION , bool tao diff --git a/templates/msg.h.em b/templates/msg.h.em index 31412c8..e940d4f 100644 --- a/templates/msg.h.em +++ b/templates/msg.h.em @@ -76,6 +76,13 @@ uint32_t _@(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer ); bool _@(msg_underscored_name)_decode(const CanardRxTransfer* transfer, @(msg_c_type)* msg); +@{coding_table = build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fields)} +@[if coding_table is not None] +#if CANARD_ENABLE_TABLE_ENCODING || CANARD_ENABLE_TABLE_DECODING +extern const CanardCodingTable _@(msg_underscored_name)_coding_table; +#endif +@[end if] + static inline uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_t* buffer #if CANARD_ENABLE_TAO_OPTION , bool tao @@ -90,6 +97,15 @@ static inline uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_ return 0; // 0-length message encodes to 0 bytes @[else] +@[if coding_table is not None] +#if CANARD_ENABLE_TABLE_ENCODING + return canardTableEncodeMessage(&_@(msg_underscored_name)_coding_table, buffer, msg +#if CANARD_ENABLE_TAO_OPTION + , tao +#endif + ); +#endif +@[end if] return _@(msg_underscored_name)_encode(msg, buffer #if CANARD_ENABLE_TAO_OPTION , tao @@ -105,6 +121,11 @@ static inline bool @(msg_underscored_name)_decode(const CanardRxTransfer* transf // all transports accurately convey a payload length of 0 bytes so any payload is an error return transfer->payload_len != 0; @[else] +@[if coding_table is not None] +#if CANARD_ENABLE_TABLE_DECODING + return canardTableDecodeMessage(&_@(msg_underscored_name)_coding_table, transfer, msg); +#endif +@[end if] return _@(msg_underscored_name)_decode(transfer, msg); @[end if] } From d8e549092f30a9623c489f9cc23df6d2d48dd73a Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 15:13:05 -0500 Subject: [PATCH 4/8] table code types containing primitive and void fields Each table has a four byte header, plus some number of entries (4 bytes). Each field consumes one entry. The maximum encoded message length in bytes, size of the C message struct in chars, and number of table entries all are limited to 65535 for table coding to be possible. --- dronecan_dsdlc_helpers.py | 42 ++++++++++++++++++++++++++++++++++++++- templates/msg.c.em | 1 + templates/msg.h.em | 12 +++++++---- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index 534b36c..78166ff 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -272,4 +272,44 @@ def mkdir_p(path): raise def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fields): - return None + if len(msg_fields) == 0: # cheaper to encode an empty message without a table + return None + if int((msg_max_bitlen+7)/8) > 65535: # too big to encode + return None + + table = _build_coding_table_core(msg_underscored_name, msg_union, msg_fields) + if table is None: return None + if len(table) > 65536: # too big to encode + return None + + # turn table description into a list of strings + table_str = [] + for kind, *params in table: + params = ", ".join(str(v) for v in params) + table_str.append(f"{kind}({params}),") + + return table_str + +def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): + if obj_union: + return None # not yet supported + + table = [] + for field in obj_fields: + t = field.type + offset = f"offsetof(struct {obj_underscored_name}, {field.name})" + field_name = t.full_name.replace('.', '_').split("[")[0] + if isinstance(t, dronecan.dsdl.parser.PrimitiveType): + if t.kind == t.KIND_BOOLEAN or t.kind == t.KIND_UNSIGNED_INT: + ts = "CANARD_TABLE_CODING_UNSIGNED" + elif t.kind == t.KIND_SIGNED_INT: + ts = "CANARD_TABLE_CODING_SIGNED" + elif t.kind == t.KIND_FLOAT: + ts = "CANARD_TABLE_CODING_FLOAT" + table.append(("CANARD_TABLE_CODING_ENTRY_PRIMITIVE", offset, ts, t.bitlen)) + elif isinstance(t, dronecan.dsdl.parser.VoidType): + table.append(("CANARD_TABLE_CODING_ENTRY_VOID", t.bitlen)) + else: + return None # unsupported type + + return table diff --git a/templates/msg.c.em b/templates/msg.c.em index 34efa3e..341ac6b 100644 --- a/templates/msg.c.em +++ b/templates/msg.c.em @@ -15,6 +15,7 @@ @[if coding_table is not None] #if CANARD_ENABLE_TABLE_ENCODING || CANARD_ENABLE_TABLE_DECODING const CanardCodingTable _@(msg_underscored_name)_coding_table = { + .max_size = @(msg_define_name.upper())_MAX_SIZE, .entries_max = @(len(coding_table))-1, .entries = { @("\n".join(" "+e for e in coding_table)) diff --git a/templates/msg.h.em b/templates/msg.h.em index e940d4f..62d7c2d 100644 --- a/templates/msg.h.em +++ b/templates/msg.h.em @@ -99,11 +99,13 @@ static inline uint32_t @(msg_underscored_name)_encode(@(msg_c_type)* msg, uint8_ @[else] @[if coding_table is not None] #if CANARD_ENABLE_TABLE_ENCODING - return canardTableEncodeMessage(&_@(msg_underscored_name)_coding_table, buffer, msg + if (sizeof(@(msg_c_type)) <= 65535) { // ensure offsets fit in table + return canardTableEncodeMessage(&_@(msg_underscored_name)_coding_table, buffer, msg #if CANARD_ENABLE_TAO_OPTION - , tao + , tao #endif - ); + ); + } #endif @[end if] return _@(msg_underscored_name)_encode(msg, buffer @@ -123,7 +125,9 @@ static inline bool @(msg_underscored_name)_decode(const CanardRxTransfer* transf @[else] @[if coding_table is not None] #if CANARD_ENABLE_TABLE_DECODING - return canardTableDecodeMessage(&_@(msg_underscored_name)_coding_table, transfer, msg); + if (sizeof(@(msg_c_type)) <= 65535) { // ensure offsets fit in table + return canardTableDecodeMessage(&_@(msg_underscored_name)_coding_table, transfer, msg); + } #endif @[end if] return _@(msg_underscored_name)_decode(transfer, msg); From 4d18b8d34454587b9143d01c24b96e45d3e93ab7 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 15:41:21 -0500 Subject: [PATCH 5/8] table code types which contain compound types There is no additional overhead for nesting. --- dronecan_dsdlc_helpers.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index 78166ff..2593141 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -284,7 +284,7 @@ def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fiel # turn table description into a list of strings table_str = [] - for kind, *params in table: + for kind, relative, *params in table: params = ", ".join(str(v) for v in params) table_str.append(f"{kind}({params}),") @@ -296,8 +296,12 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): table = [] for field in obj_fields: - t = field.type - offset = f"offsetof(struct {obj_underscored_name}, {field.name})" + if isinstance(field, dronecan.dsdl.parser.Type): + t = field + offset = 0 + else: + t = field.type + offset = f"offsetof(struct {obj_underscored_name}, {field.name})" field_name = t.full_name.replace('.', '_').split("[")[0] if isinstance(t, dronecan.dsdl.parser.PrimitiveType): if t.kind == t.KIND_BOOLEAN or t.kind == t.KIND_UNSIGNED_INT: @@ -306,9 +310,19 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): ts = "CANARD_TABLE_CODING_SIGNED" elif t.kind == t.KIND_FLOAT: ts = "CANARD_TABLE_CODING_FLOAT" - table.append(("CANARD_TABLE_CODING_ENTRY_PRIMITIVE", offset, ts, t.bitlen)) + table.append(("CANARD_TABLE_CODING_ENTRY_PRIMITIVE", False, offset, ts, t.bitlen)) elif isinstance(t, dronecan.dsdl.parser.VoidType): - table.append(("CANARD_TABLE_CODING_ENTRY_VOID", t.bitlen)) + table.append(("CANARD_TABLE_CODING_ENTRY_VOID", True, t.bitlen)) + elif isinstance(t, dronecan.dsdl.parser.CompoundType): + sub = _build_coding_table_core(field_name, t.union, t.fields) # build the compound table + if sub is None: return None + + # prepend offset to each entry that's not relative + for s_kind, s_relative, *s_params in sub: + if not s_relative: # not relative, needs offset adjusted + table.append((s_kind, s_relative, f"{offset}+{s_params[0]}", *s_params[1:])) + else: + table.append((s_kind, s_relative, *s_params)) else: return None # unsupported type From 060a649628c930970d679ecd6278aac68c029a5e Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 16:28:38 -0500 Subject: [PATCH 6/8] table code types containing static arrays Static arrays have a 2-entry header (8 bytes), plus the entries describing the contents of the array. Array contents cannot consist of more than 256 entries (including header entries and entries describing nested types). The array length cannot exceed 65535. --- dronecan_dsdlc_helpers.py | 41 ++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index 2593141..cdaab2f 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -285,8 +285,12 @@ def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fiel # turn table description into a list of strings table_str = [] for kind, relative, *params in table: - params = ", ".join(str(v) for v in params) - table_str.append(f"{kind}({params}),") + if kind is None: + # number of list items must exactly equal number of entries! + table_str.append("// auxiliary entry") + else: + params = ", ".join(str(v) for v in params) + table_str.append(f"{kind}({params}),") return table_str @@ -319,10 +323,37 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): # prepend offset to each entry that's not relative for s_kind, s_relative, *s_params in sub: - if not s_relative: # not relative, needs offset adjusted - table.append((s_kind, s_relative, f"{offset}+{s_params[0]}", *s_params[1:])) + if s_kind is None: + table.append((None, None)) + else: + if not s_relative: # not relative, needs offset adjusted + table.append((s_kind, s_relative, f"{offset}+{s_params[0]}", *s_params[1:])) + else: + table.append((s_kind, s_relative, *s_params)) + elif isinstance(t, dronecan.dsdl.parser.ArrayType) and t.mode == t.MODE_STATIC: + if t.max_size > 65535: # too big to encode + return None + + if isinstance(t.value_type, dronecan.dsdl.parser.CompoundType): + sub = _build_coding_table_core(field_name, t.value_type.union, t.value_type.fields) + else: + sub = _build_coding_table_core(field_name, False, [t.value_type]) + if sub is None: return None + if len(sub) == 0: # array of empty objects??? + assert False + if len(sub) > 256: # too many entries to encode + return None + + table.append(("CANARD_TABLE_CODING_ENTRIES_ARRAY_STATIC", False, + offset, len(sub), f"sizeof({dronecan_type_to_ctype(t.value_type)})", t.max_size + )) + table.append((None, None)) # dummy auxiliary entry for accurate entry count + for s_kind, s_relative, *s_params in sub: + if s_kind is None: + table.append((None, None)) else: - table.append((s_kind, s_relative, *s_params)) + # all entries must have a relative offset + table.append((s_kind, True, *s_params)) else: return None # unsupported type From d6d07f92941c37d1e1e6aded0998c05694368e68 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 18:01:44 -0500 Subject: [PATCH 7/8] table code dynamic arrays, including tail array optimization Dynamic arrays have a 3-entry header (12 bytes), plus the entries describing the contents of the array. Array contents cannot consist of more than 256 entries (including header entries and entries describing nested types). The maximum array length cannot exceed 65535. --- dronecan_dsdlc_helpers.py | 42 ++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index cdaab2f..2064b5e 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -277,7 +277,7 @@ def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fiel if int((msg_max_bitlen+7)/8) > 65535: # too big to encode return None - table = _build_coding_table_core(msg_underscored_name, msg_union, msg_fields) + table = _build_coding_table_core(msg_underscored_name, msg_union, msg_fields, True) if table is None: return None if len(table) > 65536: # too big to encode return None @@ -294,12 +294,13 @@ def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fiel return table_str -def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): +def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields, tao): if obj_union: return None # not yet supported table = [] - for field in obj_fields: + for field_idx, field in enumerate(obj_fields): + tao_eligible = tao and field_idx == len(obj_fields)-1 if isinstance(field, dronecan.dsdl.parser.Type): t = field offset = 0 @@ -318,7 +319,7 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): elif isinstance(t, dronecan.dsdl.parser.VoidType): table.append(("CANARD_TABLE_CODING_ENTRY_VOID", True, t.bitlen)) elif isinstance(t, dronecan.dsdl.parser.CompoundType): - sub = _build_coding_table_core(field_name, t.union, t.fields) # build the compound table + sub = _build_coding_table_core(field_name, t.union, t.fields, tao_eligible) # build the compound table if sub is None: return None # prepend offset to each entry that's not relative @@ -327,27 +328,44 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): table.append((None, None)) else: if not s_relative: # not relative, needs offset adjusted + if s_kind.startswith("CANARD_TABLE_CODING_ENTRIES_ARRAY_DYNAMIC"): # has a second offset + s_params = (*s_params[:-1], f"{offset}+{s_params[-1]}") table.append((s_kind, s_relative, f"{offset}+{s_params[0]}", *s_params[1:])) else: table.append((s_kind, s_relative, *s_params)) - elif isinstance(t, dronecan.dsdl.parser.ArrayType) and t.mode == t.MODE_STATIC: + elif isinstance(t, dronecan.dsdl.parser.ArrayType): if t.max_size > 65535: # too big to encode return None + use_tao = t.mode == t.MODE_DYNAMIC and tao_eligible and t.value_type.get_min_bitlen() >= 8 + if isinstance(t.value_type, dronecan.dsdl.parser.CompoundType): - sub = _build_coding_table_core(field_name, t.value_type.union, t.value_type.fields) + sub = _build_coding_table_core(field_name, t.value_type.union, t.value_type.fields, tao_eligible and not use_tao) else: - sub = _build_coding_table_core(field_name, False, [t.value_type]) + sub = _build_coding_table_core(field_name, False, [t.value_type], tao_eligible and not use_tao) if sub is None: return None if len(sub) == 0: # array of empty objects??? assert False if len(sub) > 256: # too many entries to encode return None - table.append(("CANARD_TABLE_CODING_ENTRIES_ARRAY_STATIC", False, - offset, len(sub), f"sizeof({dronecan_type_to_ctype(t.value_type)})", t.max_size - )) - table.append((None, None)) # dummy auxiliary entry for accurate entry count + # entries for the array itself + if t.mode == t.MODE_STATIC: + table.append(("CANARD_TABLE_CODING_ENTRIES_ARRAY_STATIC", False, + offset, len(sub), f"sizeof({dronecan_type_to_ctype(t.value_type)})", t.max_size + )) + table.append((None, None)) # dummy auxiliary entry for accurate entry count + elif t.mode == t.MODE_DYNAMIC: + # same cdef as field_cdef + cdef = 'struct { uint%u_t len; %s data[%u]; }' % (c_int_type_bitlen(array_len_field_bitlen(t)), dronecan_type_to_ctype(t.value_type), t.max_size) + table.append((f"CANARD_TABLE_CODING_ENTRIES_ARRAY_DYNAMIC", False, + f"{offset}+offsetof({cdef}, data)", 1 if use_tao else 0, len(sub), f"sizeof({dronecan_type_to_ctype(t.value_type)})", t.max_size, + t.max_size.bit_length(), f"{offset}+offsetof({cdef}, len)" + )) + table.extend(((None, None),)*2) # dummy auxiliary entries for accurate entry count + else: + assert False # unknown type + for s_kind, s_relative, *s_params in sub: if s_kind is None: table.append((None, None)) @@ -355,6 +373,6 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields): # all entries must have a relative offset table.append((s_kind, True, *s_params)) else: - return None # unsupported type + assert False # unknown type return table From a01de6c6faec5519a767e26b004105599b701852 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Sat, 20 Jul 2024 22:05:27 -0500 Subject: [PATCH 8/8] table code types containing unions Unions have a 2-entry header (8 bytes), plus 1 entry for each field, plus the entries describing each field. The maximum number of fields is 255. Each field cannot consist of more than 255 entries (including header entries and entries describing nested types). This completes the list of DSDL types. --- dronecan_dsdlc_helpers.py | 44 +++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/dronecan_dsdlc_helpers.py b/dronecan_dsdlc_helpers.py index 2064b5e..775c473 100644 --- a/dronecan_dsdlc_helpers.py +++ b/dronecan_dsdlc_helpers.py @@ -295,10 +295,45 @@ def build_coding_table(msg_underscored_name, msg_union, msg_max_bitlen, msg_fiel return table_str def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields, tao): - if obj_union: - return None # not yet supported - table = [] + + if obj_union: # unions are a special case + if len(obj_fields) == 0 or len(obj_fields) > 255: # out of range to encode + return None + + # entries to describe the union + table.append(("CANARD_TABLE_CODING_ENTRIES_UNION", False, + f"offsetof(struct {obj_underscored_name}, {obj_fields[0].name})", len(obj_fields), + union_msg_tag_bitlen_from_num_fields(len(obj_fields)), + f"offsetof(struct {obj_underscored_name}, union_tag)" + )) + table.append((None, None)) # dummy auxiliary entry for accurate entry count + + field_entries = [] + for field in obj_fields: + t = field.type + + if isinstance(t, dronecan.dsdl.parser.CompoundType): + sub = _build_coding_table_core(field.name, t.union, t.fields, tao) + else: + sub = _build_coding_table_core(field.name, False, [t], tao) + if sub is None: return None + if len(sub) > 255: # too many entries to encode + return None + + # entry to describe this field + table.append(("CANARD_TABLE_CODING_ENTRY_UNION_FIELD", True, len(sub))) + + for s_kind, s_relative, *s_params in sub: + if s_kind is None: + field_entries.append((None, None)) + else: + # all entries must have a relative offset + field_entries.append((s_kind, True, *s_params)) + + table.extend(field_entries) + return table + for field_idx, field in enumerate(obj_fields): tao_eligible = tao and field_idx == len(obj_fields)-1 if isinstance(field, dronecan.dsdl.parser.Type): @@ -328,7 +363,8 @@ def _build_coding_table_core(obj_underscored_name, obj_union, obj_fields, tao): table.append((None, None)) else: if not s_relative: # not relative, needs offset adjusted - if s_kind.startswith("CANARD_TABLE_CODING_ENTRIES_ARRAY_DYNAMIC"): # has a second offset + if s_kind.startswith("CANARD_TABLE_CODING_ENTRIES_ARRAY_DYNAMIC") \ + or s_kind == "CANARD_TABLE_CODING_ENTRIES_UNION": # has a second offset s_params = (*s_params[:-1], f"{offset}+{s_params[-1]}") table.append((s_kind, s_relative, f"{offset}+{s_params[0]}", *s_params[1:])) else: