test: Make esp32 build actually try to instantiate tox.

It doesn't work, because esp32 has too little RAM (320KB). DHT is a
240KB struct, so even just allocating that immediately fails. We'll need
to think carefully about trimming that if we ever want this to work on
embedded devices.

.github/workflows/docker.yml

@@ -129,19 +129,14 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build toxchat/c-toxcore:sources
-        uses: docker/build-push-action@v2
-        with:
-          file: other/docker/sources/Dockerfile
-          tags: toxchat/c-toxcore:sources
       - name: Build and push
         uses: docker/build-push-action@v2
         with:
           file: other/docker/esp32/Dockerfile
-          push: ${{ github.event_name == 'push' }}
           tags: toxchat/c-toxcore:esp32
           cache-from: type=registry,ref=toxchat/c-toxcore:esp32
           cache-to: type=inline
+          push: ${{ github.event_name == 'push' }}
 
   docker-win32:
     runs-on: ubuntu-latest

auto_tests/onion_test.c

@@ -53,7 +53,7 @@ static int handle_test_1(void *object, const IP_Port *source, const uint8_t *pac
     res_packet[0] = NET_PACKET_ANNOUNCE_RESPONSE;
     memcpy(res_packet + 1, res_message, sizeof(res_message));
 
-    if (send_onion_response(onion->net, source, res_packet, sizeof(res_packet),
+    if (send_onion_response(onion->log, onion->net, source, res_packet, sizeof(res_packet),
                             packet + sizeof(res_packet)) == -1) {
         return 1;
     }

other/bootstrap_daemon/docker/tox-bootstrapd.sha256

@@ -1 +1 @@
-1608abb52cd16775216d01d06569e6437d86ed11e9fc8dc6dc9c1c28668ccd8b  /usr/local/bin/tox-bootstrapd
+bc9e2100e1b8951932a4b82b322983e0b40ba9bbe8b505769aa974b8b4116247  /usr/local/bin/tox-bootstrapd

other/docker/esp32/Dockerfile

@@ -1,48 +1,32 @@
-FROM toxchat/c-toxcore:sources AS src
-FROM ubuntu:18.04
+FROM mluis/qemu-esp32:latest
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV IDF_TARGET=esp32
 
-RUN apt-get update && \
- DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
- bison \
- ccache \
- cmake \
- flex \
- git \
- gperf \
- libncurses-dev \
- ninja-build \
- python \
- python-cryptography \
- python-future \
- python-pip \
- python-pyparsing \
- python-serial \
- python-setuptools \
- wget \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
+WORKDIR /root/toxcore
 
-ENV ESP32_TARBALL=xtensa-esp32-elf-linux64-1.22.0-80-g6c4433a-5.2.0 \
-    IDF_PATH="/root/esp/esp-idf" \
-    PATH="/root/esp/esp-idf/tools:/root/esp/xtensa-esp32-elf/bin:$PATH"
-WORKDIR /root/esp
-RUN wget -q https://dl.espressif.com/dl/$ESP32_TARBALL.tar.gz \
- && tar zxf $ESP32_TARBALL.tar.gz \
- && rm -f $ESP32_TARBALL.tar.gz \
- && git clone -b v3.3 --recursive --depth=1 --shallow-submodules https://github.com/espressif/esp-idf
+# Build an initial bootstrap hello world just to compile libsodium and other
+# system level dependencies.
+COPY other/docker/esp32/sdkconfig \
+     other/docker/esp32/CMakeLists.txt \
+     /root/toxcore/
+COPY other/docker/esp32/bootstrap/ \
+     /root/toxcore/main/
+RUN . /root/esp/esp-idf/export.sh && idf.py build
 
-# Build a hello world first, so the OS and libsodium etc. are compiled.
-WORKDIR /root/esp/toxcore
-COPY other/docker/esp32/CMakeLists.txt /root/esp/toxcore/
-COPY other/docker/esp32/hello/ /root/esp/toxcore/main/
-RUN idf.py build
-
-# Then copy over the actual toxcore sources and build those.
-COPY --from=src /src/third_party/cmp/ /root/esp/toxcore/main/third_party/cmp/
-COPY --from=src /src/toxencryptsave/defines.h /root/esp/toxcore/main/toxencryptsave/
-COPY --from=src /src/toxcore/ /root/esp/toxcore/main/toxcore/
-COPY other/docker/esp32/toxcore/CMakeLists.txt /root/esp/toxcore/main/
-COPY other/docker/esp32/toxcore/toxcore_main.cc /root/esp/toxcore/main/other/docker/esp32/main/
-RUN idf.py build
+# Copy over toxcore sources and build those.
+COPY third_party/cmp/ /root/toxcore/main/third_party/cmp/
+COPY toxencryptsave/defines.h /root/toxcore/main/toxencryptsave/
+COPY toxcore/ /root/toxcore/main/toxcore/
+COPY other/docker/esp32/main/CMakeLists.txt \
+     /root/toxcore/main/
+COPY other/docker/esp32/main/*.cc \
+     other/docker/esp32/main/*.h \
+     /root/toxcore/main/other/docker/esp32/main/
+RUN touch /root/toxcore/main/CMakeLists.txt \
+ && . /root/esp/esp-idf/export.sh && idf.py build
 RUN ls -lh build/toxcore.bin \
- && shasum build/toxcore.bin
+ && sha512sum build/toxcore.bin
+RUN /root/flash.sh build/toxcore.bin
+
+COPY other/docker/esp32/qemu-test /root/toxcore/
+RUN ["/root/toxcore/qemu-test"]

other/docker/esp32/bootstrap/CMakeLists.txt

@@ -0,0 +1,4 @@
+idf_component_register(
+  SRCS hello_main.cc
+  INCLUDE_DIRS "."
+  REQUIRES esp_netif lwip)

other/docker/esp32/bootstrap/hello_main.cc

@@ -0,0 +1,7 @@
+#include <stdio.h>
+
+// Bootstrap main. Only writes hello world. See ../main/* for the real thing.
+extern "C" void app_main(void)
+{
+    printf("Hello world!\n");
+}

other/docker/esp32/bootstrap/idf_component.yml

@@ -0,0 +1,2 @@
+dependencies:
+  espressif/libsodium: "==1.0.20"

other/docker/esp32/main/CMakeLists.txt

@@ -0,0 +1,16 @@
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-field-initializers -Wno-format -DMIN_LOGGER_LEVEL=LOGGER_LEVEL_TRACE")
+
+file(GLOB toxcore_SRCS "toxcore/*.[ch]" "toxcore/*/*.[ch]")
+set(COMPONENT_SRCS
+  ${toxcore_SRCS}
+  other/docker/esp32/main/app_main.cc
+  other/docker/esp32/main/tox_main.cc
+  other/docker/esp32/main/tox_main.h
+  third_party/cmp/cmp.c
+  third_party/cmp/cmp.h
+  toxencryptsave/defines.h)
+
+idf_component_register(
+  SRCS ${COMPONENT_SRCS}
+  INCLUDE_DIRS "."
+  REQUIRES esp_eth esp_netif lwip)

other/docker/esp32/main/app_main.cc

@@ -0,0 +1,66 @@
+#include <stdio.h>
+#include <time.h>
+
+#include <esp_event.h>
+#include <esp_eth.h>
+#include <esp_log.h>
+#include <esp_netif.h>
+#include <esp_netif_sntp.h>
+
+#include "tox_main.h"
+
+static const char *MAIN_TAG = "app_main";
+
+static esp_eth_handle_t eth_handle = nullptr;
+static esp_netif_t *eth_netif = nullptr;
+
+static void event_handler(void *arg, esp_event_base_t event_base,
+                          int32_t event_id, void *event_data)
+{
+    if (event_base == ETH_EVENT) {
+        if (event_id == ETHERNET_EVENT_START) return;
+        if (event_id == ETHERNET_EVENT_STOP) return;
+    }
+    if (event_base == IP_EVENT)
+        if (event_id == IP_EVENT_ETH_GOT_IP)
+		return;
+}
+
+static void register_ethernet(void)
+{
+    ESP_ERROR_CHECK(esp_netif_init());
+    ESP_ERROR_CHECK(esp_event_loop_create_default());
+    esp_netif_config_t cfg = ESP_NETIF_DEFAULT_ETH();
+    eth_netif = esp_netif_new(&cfg);
+
+    eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG();
+    eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG();
+    esp_eth_mac_t *mac = esp_eth_mac_new_openeth(&mac_config);
+
+    esp_eth_phy_t *phy = esp_eth_phy_new_dp83848(&phy_config);
+
+    esp_eth_config_t config = ETH_DEFAULT_CONFIG(mac, phy);
+    ESP_ERROR_CHECK(esp_eth_driver_install(&config, &eth_handle));
+    ESP_ERROR_CHECK(esp_netif_attach(eth_netif, esp_eth_new_netif_glue(eth_handle)));
+    ESP_ERROR_CHECK(esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, &event_handler, NULL));
+    ESP_ERROR_CHECK(esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, &event_handler, NULL));
+    ESP_ERROR_CHECK(esp_eth_start(eth_handle));
+}
+
+// Does all the esp32-specific init before running generic tox code.
+extern "C" void app_main(void)
+{
+    register_ethernet();
+
+    esp_sntp_config_t config = ESP_NETIF_SNTP_DEFAULT_CONFIG("pool.ntp.org");
+    ESP_ERROR_CHECK(esp_netif_sntp_init(&config));
+
+    if (esp_netif_sntp_sync_wait(pdMS_TO_TICKS(10000)) != ESP_OK) {
+        ESP_LOGE(MAIN_TAG, "failed to update system time within 10s timeout");
+        return;
+    }
+
+    ESP_LOGI(MAIN_TAG, "time is updated: %lld", time(nullptr));
+
+    tox_main();
+}

other/docker/esp32/main/tox_main.cc

@@ -0,0 +1,121 @@
+#include "../main/tox_main.h"
+
+#include <assert.h>
+#include <memory>
+#include <stdio.h>
+#include <time.h>
+
+#include "../../../../toxcore/ccompat.h"
+#include "../../../../toxcore/tox.h"
+#include "../../../../toxcore/tox_events.h"
+
+static char tox_log_level_name(Tox_Log_Level level)
+{
+    switch (level) {
+        case TOX_LOG_LEVEL_TRACE: return 'T';
+        case TOX_LOG_LEVEL_DEBUG: return 'D';
+        case TOX_LOG_LEVEL_INFO: return 'I';
+        case TOX_LOG_LEVEL_WARNING: return 'W';
+        case TOX_LOG_LEVEL_ERROR: return 'E';
+    }
+
+    return '?';
+}
+
+static const char *tox_err_new_name(Tox_Err_New err)
+{
+    switch (err) {
+        case TOX_ERR_NEW_OK: return "OK";
+        case TOX_ERR_NEW_NULL: return "NULL";
+        case TOX_ERR_NEW_MALLOC: return "MALLOC";
+        case TOX_ERR_NEW_PORT_ALLOC: return "PORT_ALLOC";
+        case TOX_ERR_NEW_PROXY_BAD_TYPE: return "PROXY_BAD_TYPE";
+        case TOX_ERR_NEW_PROXY_BAD_HOST: return "PROXY_BAD_HOST";
+        case TOX_ERR_NEW_PROXY_BAD_PORT: return "PROXY_BAD_PORT";
+        case TOX_ERR_NEW_PROXY_NOT_FOUND: return "PROXY_NOT_FOUND";
+        case TOX_ERR_NEW_LOAD_ENCRYPTED: return "LOAD_ENCRYPTED";
+        case TOX_ERR_NEW_LOAD_BAD_FORMAT: return "LOAD_BAD_FORMAT";
+    }
+
+    return "<unknown>";
+}
+
+static const char *color(int index)
+{
+    switch (index) {
+        case 0: return "\033[35m";
+        case 1: return "\033[36m";
+    }
+
+    return "\033[0m";
+}
+
+static tox_log_cb log_handler;
+static void log_handler(Tox *tox, Tox_Log_Level level, const char *file, uint32_t line, const char *func, const char *msg, void *user_data)
+{
+    const int *index = (const int *)user_data;
+    const uint16_t udp_port = tox_self_get_udp_port(tox, nullptr);
+    printf("%s#%d (:%d) [%c] %s:%u(%s): %s\n", color(*index), *index, udp_port,
+           tox_log_level_name(level), file, (unsigned int)line, func, msg);
+}
+
+using Tox_Options_Ptr = std::unique_ptr<Tox_Options, void(*)(Tox_Options *)>;
+using Tox_Ptr = std::unique_ptr<Tox, void(*)(Tox *)>;
+
+void tox_main()
+{
+    printf("Hello Tox!\n");
+
+    Tox_Options_Ptr opts(tox_options_new(nullptr), tox_options_free);
+    assert(opts != nullptr);
+
+    tox_options_set_ipv6_enabled(opts.get(), false);
+    tox_options_set_local_discovery_enabled(opts.get(), false);
+
+    tox_options_set_log_callback(opts.get(), log_handler);
+
+    Tox_Err_New err;
+
+    int index[] = {0, 1};
+
+    tox_options_set_log_user_data(opts.get(), &index[0]);
+    Tox_Ptr tox0(tox_new(opts.get(), &err), tox_kill);
+    printf("tox_new(#0): %p\n", (void *)tox0.get());
+
+    if (err != TOX_ERR_NEW_OK) {
+        printf("tox_new(#0): %s\n", tox_err_new_name(err));
+        return;
+    }
+
+    tox_options_set_log_user_data(opts.get(), &index[1]);
+    Tox_Ptr tox1(tox_new(opts.get(), &err), tox_kill);
+    printf("tox_new(#1): %p\n", (void *)tox1.get());
+
+    if (err != TOX_ERR_NEW_OK) {
+        printf("tox_new(#1): %s\n", tox_err_new_name(err));
+        return;
+    }
+
+    uint8_t pk[TOX_PUBLIC_KEY_SIZE];
+    tox_self_get_dht_id(tox0.get(), pk);
+    tox_bootstrap(tox1.get(), "localhost", tox_self_get_udp_port(tox0.get(), nullptr), pk, nullptr);
+
+#if 0
+    tox_self_get_public_key(tox0.get(), pk);
+    tox_friend_add_norequest(tox1.get(), pk, nullptr);
+
+    tox_self_get_public_key(tox1.get(), pk);
+    tox_friend_add_norequest(tox0.get(), pk, nullptr);
+#endif
+
+    printf("bootstrapping and connecting 2 toxes\n");
+
+    while (tox_self_get_connection_status(tox1.get()) == TOX_CONNECTION_NONE ||
+            tox_self_get_connection_status(tox0.get()) == TOX_CONNECTION_NONE) {
+        tox_events_free(tox_events_iterate(tox0.get(), true, nullptr));
+        tox_events_free(tox_events_iterate(tox1.get(), true, nullptr));
+
+        usleep(tox_iteration_interval(tox0.get()) * 1000);
+        usleep(250);  // a bit less noise in the log
+    }
+}

other/docker/esp32/main/tox_main.h

@@ -0,0 +1,6 @@
+#ifndef TOX_MAIN_H
+#define TOX_MAIN_H
+
+void tox_main();
+
+#endif // TOX_MAIN_H

other/docker/esp32/qemu-test

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eu
+
+qemu-system-xtensa \
+  -nographic \
+  -M esp32 \
+  -m 4 \
+  -drive file=flash.bin,if=mtd,format=raw \
+  -nic user,model=open_eth,hostfwd=tcp::80-:80 \
+  -s \
+  | tee qemu.log &
+
+echo "Waiting for program to complete"
+while ! grep 'Returned from app_main' qemu.log > /dev/null; do
+  sleep 1
+done

other/docker/esp32/run

@@ -1,4 +1,3 @@
 #!/bin/sh
 
-docker build -t toxchat/c-toxcore:sources -f other/docker/sources/Dockerfile .
 docker build -t toxchat/c-toxcore:esp32 -f other/docker/esp32/Dockerfile .