ethos: race condition seems to cause packet loss

[miri64]

Description

I did not finish my final analysis yet, but there seems to be a race condition in the RIOT-side of ethos, that is the cause of packet loss over the wire: I was able to pinpoint loss in ethos to this if not being taken

RIOT/drivers/ethos/ethos.c

Lines 94 to 98 in 68e1d29

	if (dev->accept_new) {
	if (tsrb_add_one(&dev->inbuf, c) == 0) {
	dev->framesize++;
	}
	}

not being called, since dev->accept_new, when barraging the ethos line (with ethos_stdio).

Steps to reproduce the issue

I used the the following patch to analyze the loss behavior:

Patch

diff --git a/dist/tools/ethos/ethos.c b/dist/tools/ethos/ethos.c
index fc4f8c94ce..ba82ee65b9 100644
--- a/dist/tools/ethos/ethos.c
+++ b/dist/tools/ethos/ethos.c
@@ -245,6 +245,8 @@ static int _serial_handle_byte(serial_t *serial, char c)
     return 0;
 }
 
+static uint16_t _last_seq_sent;
+
 static void _write_escaped(int fd, char* buf, ssize_t n)
 {
     /*
@@ -260,11 +262,36 @@ static void _write_escaped(int fd, char* buf, ssize_t n)
      * call write() on the buffer instead of one char at a time */
     uint8_t out[SERIAL_BUFFER_SIZE];
     size_t buffered = 0;
+    char *orig = buf;
+    int icmpv6 = 0, echo = 0;
+    unsigned char seq_upper = 0;
 
     while(n--) {
         unsigned char c = *buf++;
+
+        if (icmpv6 && echo && (buf - orig) == 61) {
+            seq_upper = c;
+        }
+        else if (icmpv6 && echo && (buf - orig) == 62) {
+            uint16_t seq = seq_upper << 8;
+
+            seq |= c;
+            if (_last_seq_sent != (seq - 1)) {
+                fprintf(stderr, "%u sent out of order\n", seq);
+            }
+            _last_seq_sent = seq;
+        }
+        else if (icmpv6 && (buf - orig) == 55 && c == 128) {
+            echo = 1;
+        }
+        else if ((buf - orig) == 21 && c == 58) {
+            icmpv6 = 1;
+        }
         if (c == LINE_FRAME_DELIMITER || c == LINE_ESC_CHAR) {
             out[buffered++] = LINE_ESC_CHAR;
+            if (icmpv6 && echo) {
+                fprintf(stderr, "Escaped %u (offset %lu) in %u\n", c, buf - orig, _last_seq_sent);
+            }
             c ^= 0x20;
             if (buffered >= SERIAL_BUFFER_SIZE) {
                 checked_write(fd, out, buffered);
@@ -497,6 +524,8 @@ int _open_connection(char *name, char* option)
     }
 }
 
+static uint16_t _last_seq_recv;
+
 int main(int argc, char *argv[])
 {
     char inbuf[MTU];
@@ -595,6 +624,15 @@ int main(int argc, char *argv[])
                 fprintf(stderr, "error reading from tap device. res=%zi\n", res);
                 continue;
             }
+            if (res >= 62 && (((uint8_t *)inbuf)[20] == 58) && (((uint8_t *)inbuf)[54] == 128)) {
+                uint16_t seq = (((uint8_t *)inbuf)[60]) << 8;
+
+                seq |= (((uint8_t *)inbuf)[61]);
+                if (_last_seq_recv != (seq - 1)) {
+                    fprintf(stderr, "%u received out of order\n", seq);
+                }
+                _last_seq_recv = seq;
+            }
             char delim = LINE_FRAME_DELIMITER;
             checked_write(serial_fd, &delim, 1);
             _write_escaped(serial_fd, inbuf, res);
diff --git a/drivers/ethos/ethos.c b/drivers/ethos/ethos.c
index 8fd83b8692..d08b5421eb 100644
--- a/drivers/ethos/ethos.c
+++ b/drivers/ethos/ethos.c
@@ -52,6 +52,7 @@ static const netdev_driver_t netdev_driver_ethos;
 
 static const uint8_t _esc_esc[] = {ETHOS_ESC_CHAR, (ETHOS_ESC_CHAR ^ 0x20)};
 static const uint8_t _esc_delim[] = {ETHOS_ESC_CHAR, (ETHOS_FRAME_DELIMITER ^ 0x20)};
+static uint16_t _req_recv = 0;
 
 void ethos_setup(ethos_t *dev, const ethos_params_t *params, uint8_t idx,
                  void *inbuf, size_t inbuf_size)
@@ -85,6 +86,8 @@ static void _reset_state(ethos_t *dev)
     dev->accept_new = true;
 }
 
+/* static uint16_t _last_seq_recv_serial; */
+
 static void _handle_char(ethos_t *dev, char c)
 {
     switch (dev->frametype) {
@@ -95,9 +98,12 @@ static void _handle_char(ethos_t *dev, char c)
                 if (tsrb_add_one(&dev->inbuf, c) == 0) {
                     dev->framesize++;
                 }
+                else {
+                    puts("!t");
+                }
             }
             else {
-                //puts("lost frame");
+                puts("!a");
                 dev->inbuf.reads = 0;
                 dev->inbuf.writes = 0;
                 _reset_state(dev);
@@ -299,10 +305,14 @@ static void _get_mac_addr(netdev_t *encdev, uint8_t* buf)
     memcpy(buf, dev->mac_addr, 6);
 }
 
+static uint16_t _last_seq_recv_netdev;
+
 static int _recv(netdev_t *netdev, void *buf, size_t len, void* info)
 {
     (void) info;
     ethos_t * dev = (ethos_t *) netdev;
+    uint8_t *buf8 = buf;
+
 
     if (buf) {
         if (len < dev->last_framesize) {
@@ -317,7 +327,16 @@ static int _recv(netdev_t *netdev, void *buf, size_t len, void* info)
             dev->last_framesize = 0;
             return -1;
         }
+        if ((len >= 62) && (buf8[20] == 58) && buf8[54] == 128) {
+            uint16_t seq = buf8[60] << 8;
 
+            seq |= buf8[61];
+            if (_last_seq_recv_netdev != (seq - 1)) {
+                printf("n;%u\n", seq);
+            }
+            _last_seq_recv_netdev = seq;
+            _req_recv++;
+        }
         dev->last_framesize = 0;
         return (int)len;
     }
@@ -333,6 +352,15 @@ static int _recv(netdev_t *netdev, void *buf, size_t len, void* info)
     }
 }
 
+void print_req_recv(void)
+{
+    printf("req_recv: %d\n", _req_recv);
+}
+
+void print_resp_sent(void)
+{
+}
+
 static int _get(netdev_t *dev, netopt_t opt, void *value, size_t max_len)
 {
     int res = 0;
diff --git a/examples/gnrc_border_router/main.c b/examples/gnrc_border_router/main.c
index 654b411a50..5bde49954d 100644
--- a/examples/gnrc_border_router/main.c
+++ b/examples/gnrc_border_router/main.c
@@ -26,6 +26,23 @@
 #define MAIN_QUEUE_SIZE     (8)
 static msg_t _main_msg_queue[MAIN_QUEUE_SIZE];
 
+extern void print_req_recv(void);
+extern void print_resp_sent(void);
+
+static int _print_req_recv(int argc, char **argv)
+{
+    (void)argc;
+    (void)argv;
+    print_req_recv();
+    print_resp_sent();
+    return 0;
+}
+
+static shell_command_t cmd[] = {
+    { "req_recv", "", _print_req_recv },
+    { NULL, NULL, NULL },
+};
+
 int main(void)
 {
     /* we need a message queue for the thread running the shell in order to
@@ -36,7 +53,7 @@ int main(void)
     /* start shell */
     puts("All up, running the shell now");
     char line_buf[SHELL_DEFAULT_BUFSIZE];
-    shell_run(NULL, line_buf, SHELL_DEFAULT_BUFSIZE);
+    shell_run(cmd, line_buf, SHELL_DEFAULT_BUFSIZE);
 
     /* should be never reached */
     return 0;

and flashed the examples/gnrc_border_router example in its default config to an iotlab-m3.

The patch introduces several diagnostic tools:

• The output Escaped X (offset Y) in Z can be ignored, it was just used to assure the issue does not stem from escaping.
• !a is printed every time in _handle_char when the if (dev->accept_new) branch is not taken,
• n;Z is printed when in the netdev recv method the echo request sequence number Z-1 was skipped
• A shell command req_recv is introduced to the examples/gnrc_border_router example that prints the number of ICMPv6 echo requests seen by the netdev recv method.

I used ifconfig to determine the link-local address of the ethos interface on the iotlab-m3 and pinged it with an interval of 10ms.

ping -i 0.01 fe80::a804:28ff:fefa:f3f8%tap0

Expected results

The number of packets transmitted by ping should equal the number returned by the patched-in req_recv command, the live diagnostic output added by the patch should not appear (except for the Escaped X (offset Y) in Z output).

Actual results

The number of packets transmitted by ping is larger than the number returned by the patched-in req_recv command, the live diagnostic output shows

!a
n;Z

where Z is an echo request sequence number that saw no predecessor, is shown for every echo request that is missing in the number returned by the patched-in req_recv command, indicating, that the echo request was lost, due to branch above not taken (!a may be shown multiple times before n;Z due to other frames lost).

Versions

Analysis was done on 770fe2f, but I guess this issue exists since the inception of ethos.

[miri64]

That the loss does not happen on the host side, can be seen by the fact, that there is no diagnostic output indicating that from the host-side tool (which the patch also adds).

[miri64]

The issue is not reproducable on nrf52840dongle with UPLINK=cdc-ecm. There, all pings of the first 1000 or so are transmitted and received (maybe of interest for @haukepetersen).

[miri64]

If I output the bytes at the "two ends of the wire" like this

diff --git a/core/include/log.h b/core/include/log.h
index d656f4ab21..05349f0314 100644
--- a/core/include/log.h
+++ b/core/include/log.h
@@ -68,7 +68,7 @@ enum {
 /**
  * @brief Default log level define
  */
-#define LOG_LEVEL LOG_INFO
+#define LOG_LEVEL LOG_NONE
 #endif
 
 /**
diff --git a/dist/tools/ethos/ethos.c b/dist/tools/ethos/ethos.c
index fc4f8c94ce..a93f1ae817 100644
--- a/dist/tools/ethos/ethos.c
+++ b/dist/tools/ethos/ethos.c
@@ -44,6 +44,8 @@
 /* Size of serial write buffer */
 #define SERIAL_BUFFER_SIZE 64
 
+static int _serial_fd;
+
 static void usage(void)
 {
     fprintf(stderr, "Usage: ethos <tap> <serial> [baudrate]\n");
@@ -52,6 +54,14 @@ static void usage(void)
 
 static void checked_write(int handle, void *buffer, int nbyte)
 {
+    if (handle == _serial_fd) {
+        for (int i = 0; i < nbyte; i++) {
+            const uint8_t c = ((uint8_t *)buffer)[i];
+            fputc((c >> 4) + '0', stderr);
+            fputc((c & 0xf) + '0', stderr);
+        }
+        fputc('\n', stderr);
+    }
     while (nbyte > 0) {
         ssize_t res = write(handle, buffer, nbyte);
         if (res <= 0) {
@@ -526,6 +536,7 @@ int main(int argc, char *argv[])
         fprintf(stderr, "Error opening serial device %s\n", argv[2]);
         return 1;
     }
+    _serial_fd = serial_fd;
 
     fd_set readfds;
     int max_fd = (serial_fd > tap_fd) ? serial_fd : tap_fd;
diff --git a/drivers/ethos/ethos.c b/drivers/ethos/ethos.c
index 8fd83b8692..c91a52403f 100644
--- a/drivers/ethos/ethos.c
+++ b/drivers/ethos/ethos.c
@@ -139,6 +139,14 @@ static void ethos_isr(void *arg, uint8_t c)
 {
     ethos_t *dev = (ethos_t *) arg;
 
+    if (c == 0x7e) {
+        fputc('\n', stdout);
+    }
+    fputc((c >> 4) + '0', stdout);
+    fputc((c & 0xf) + '0', stdout);
+    if (c == 0x7e) {
+        fputc('\n', stdout);
+    }
     switch (dev->state) {
         case WAIT_FRAMESTART:
             if (c == ETHOS_FRAME_DELIMITER) {
diff --git a/examples/gnrc_border_router/Makefile.board.dep b/examples/gnrc_border_router/Makefile.board.dep
index 12f755ceb4..40a7fe5d88 100644
--- a/examples/gnrc_border_router/Makefile.board.dep
+++ b/examples/gnrc_border_router/Makefile.board.dep
@@ -3,7 +3,7 @@ ifeq (,$(filter native,$(BOARD)))
   ifeq (slip,$(UPLINK))
     USEMODULE += slipdev_stdio
   else ifeq (ethos,$(UPLINK))
-    USEMODULE += stdio_ethos
+    USEMODULE += ethos
   else ifeq (wifi,$(UPLINK))
     ifneq (,$(filter esp32 esp8266,$(CPU)))
       USEMODULE += esp_wifi
diff --git a/examples/gnrc_border_router/Makefile.ethos.conf b/examples/gnrc_border_router/Makefile.ethos.conf
index 9a7d1484cd..bbabde3835 100644
--- a/examples/gnrc_border_router/Makefile.ethos.conf
+++ b/examples/gnrc_border_router/Makefile.ethos.conf
@@ -1,4 +1,4 @@
-CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE)
+CFLAGS += -DETHOS_BAUDRATE=$(ETHOS_BAUDRATE) -DETHOS_UART="UART_DEV(1)"
 
 STATIC_ROUTES ?= 1
 
@@ -9,4 +9,4 @@ endif
 # Configure terminal parameters
 TERMDEPS += host-tools
 TERMPROG ?= sudo sh $(RIOTTOOLS)/ethos/start_network.sh
-TERMFLAGS ?= $(FLAGS_EXTRAS) $(PORT) $(TAP) $(IPV6_PREFIX) $(ETHOS_BAUDRATE)
+TERMFLAGS ?= $(FLAGS_EXTRAS) /dev/ttyUSB0 $(TAP) $(IPV6_PREFIX) $(ETHOS_BAUDRATE)

and use ethos over an external FTDI adapter (I first made sure that the issue persists as described in OP), I get after some manual work the following diff:

--- ethos-host-bytes.txt	2021-11-23 18:49:07.722777135 +0100
+++ ethos-node-bytes.txt	2021-11-23 18:49:04.852746903 +0100
@@ -7,25 +7,25 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f500390001
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f400390002
+a600e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f400390002
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f300390003
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f200390004
+a6efe2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f200390004
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f100390005
+a66b88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f100390005
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f000390006
+a6efbc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0f000390006
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ef00390007
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ee00390008
+a6becbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ee00390008
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ed00390009
@@ -34,31 +34,31 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ec0039000a
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0eb0039000b
+a688bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0eb0039000b
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ea0039000c
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e90039000d
+a67d86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e90039000d
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e80039000e
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e70039000f
+a6887d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e70039000f
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e600390010
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e500390011
+a6bb7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e500390011
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e400390012
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e300390013
+a6bcf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e300390013
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e200390014
@@ -67,13 +67,13 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e100390015
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e000390016
+a6becbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0e000390016
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0df00390017
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0de00390018
+a6cbdd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0de00390018
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0dd00390019
@@ -82,7 +82,7 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0dc0039001a
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0db0039001b
+a65d86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0db0039001b
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0da0039001c
@@ -91,7 +91,7 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d90039001d
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d80039001e
+a6bb5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d80039001e
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d70039001f
@@ -100,13 +100,13 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d600390020
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d500390021
+a6868400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d500390021
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d400390022
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d300390023
+a6e2cbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d300390023
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d200390024
@@ -115,7 +115,7 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d100390025
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d000390026
+a6bcf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0d000390026
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cf00390027
@@ -124,25 +124,25 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ce00390028
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cd00390029
+a6efbc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cd00390029
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cc0039002a
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cb0039002b
+a6becbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0cb0039002b
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0ca0039002c
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c90039002d
+a6e2f54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c90039002d
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c80039002e
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c70039002f
+a6efbc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c70039002f
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c600390030
@@ -151,7 +151,7 @@
 a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c500390031
 7e
 7e
-a66bef88bbbee2bc7d5dcbf54f86dd6007b38400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c400390032
+a6868400083a40fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbe8000e0c400390032
 7e
 7e
 a66bef88bbbee2bc7d5dcbf54f86dd600dc35000161140fe80000000000000e0bc7d5dfffecbf54ffe80000000000000a46beffffe88bbbeb1df30390016b78a

The manual work enticed: aligning newlines and mapping the ASCII-based hexadecimal output to the traditional one:

ASCII hex	alpha hex
:	a
;	b
<	c
=	d
>	e
?	f

So I definitely use bytes (at with this approach), but maybe this is due to the interrupt being blocked, since I print stuff in it 😅.

[miri64]

and use ethos over an external FTDI adapter (I first made sure that the issue persists as described in OP.

I used a samr21-xpro to do that BTW..., not an IoT-LAB.

[miri64]

The issue is not reproducable on nrf52840dongle with UPLINK=cdc-ecm. There, all pings of the first 1000 or so are transmitted and received (maybe of interest for @haukepetersen).

Same goes for samr21-xpro and iotlab-m3 with UPLINK=slip. On iotlab-m3, I see a reproducible loss of 0.1% though.

[miri64]

So I definitely use bytes (at with this approach), but maybe this is due to the interrupt being blocked, since I print stuff in it 😅.

This:

Same goes for samr21-xpro and iotlab-m3 with UPLINK=slip. On iotlab-m3, I see a reproducible loss of 0.1% though.

let's me think, that the latter of the initial quote is rather the case. Especially, since I multiplexed stdio over SLIP in that trial.

[miri64]

The race condition is fixed in #17265.