Build and installer fixes (#2429)

**Host**

- Fix host --help option
- Add --loopcount emulator option

**Build**

- Update FixPath to handle spaces in path
- Fix COMPONENT_SEARCH_DIRS for Windows
- Fix appcode object path resolution, ensure object path is relative

**Installer**

- `all` option missing rp2040
- Remove `doc` option from `all`

**CI testing**

- Revise to simplify use by external libraries

Co-authored-by: Slavey Karadzhov <[email protected]>

Sming/Arch/Host/Components/hostlib/options.h

@@ -32,9 +32,9 @@
 	XX(uart, required_argument, "Enable UART server", "PORT", "Which UART number to enable",                           \
 	   "e.g. --uart=0 --uart=1 enable servers for UART0, UART1\0")                                                     \
 	XX(device, required_argument, "Set device for uart", "DEVICE", "Optionally map uart to device",                    \
-	   "e.g. --uart=0 --device=/dev/ttyUSB0")                                                                          \
+	   "e.g. --uart=0 --device=/dev/ttyUSB0\0")                                                                        \
 	XX(baud, required_argument, "Set baud rate for UART", "BAUD", "Requires --device argument",                        \
-	   "e.g. --uart=0 --device=/dev/ttyUSB0 --baud=115200")                                                            \
+	   "e.g. --uart=0 --device=/dev/ttyUSB0 --baud=115200\0")                                                          \
 	XX(portbase, required_argument, "Specify base port number for UART socket servers", "PORT", "IP port number",      \
 	   nullptr)                                                                                                        \
 	XX(ifname, required_argument, "Specify network interface", "NAME", "Network interface to use (e.g. tap0)",         \
@@ -52,6 +52,8 @@
 	XX(flashsize, required_argument, "Change default flash size if file doesn't exist", "SIZE",                        \
 	   "Size of flash in bytes (e.g. 512K, 524288, 0x80000)", nullptr)                                                 \
 	XX(initonly, no_argument, "Initialise only, do not start Sming", nullptr, nullptr, nullptr)                        \
+	XX(loopcount, required_argument, "Run Sming loop a fixed number of times then exit", nullptr, nullptr,             \
+	   "Useful for running samples in CI\0")                                                                           \
 	XX(nonet, no_argument, "Skip network initialisation", nullptr, nullptr, nullptr)                                   \
 	XX(debug, required_argument, "Set debug verbosity", "LEVEL", "Maximum debug message level to print",               \
 	   "0 = errors only, 1 = +warnings, 2 = +info\0")

Sming/Arch/Host/Components/hostlib/startup.cpp

@@ -134,6 +134,7 @@ int main(int argc, char* argv[])
 		int pause;
 		int exitpause;
 		bool initonly;
+		int loopcount;
 		bool enable_network;
 		UartServer::Config uart;
 		FlashmemConfig flash;
@@ -244,6 +245,10 @@ int main(int argc, char* argv[])
 			config.initonly = true;
 			break;
 
+		case opt_loopcount:
+			config.loopcount = atoi(arg);
+			break;
+
 		case opt_nonet:
 			config.enable_network = false;
 			break;
@@ -309,6 +314,14 @@ int main(int argc, char* argv[])
 #endif
 		while(!done) {
 			host_main_loop();
+			if(config.loopcount == 0) {
+				continue;
+			}
+			--config.loopcount;
+			if(config.loopcount == 0) {
+				host_debug_i("Reached requested loop count limit: exiting");
+				break;
+			}
 		}
 
 		host_debug_i(">> Normal Exit <<\n");

Sming/Arch/Host/Tools/ci/install.cmd

@@ -1,18 +1,3 @@
 REM Host install.cmd
 
-call :install "c:\tools\doxygen" doxygen-1.9.1.windows.bin.zip
-call :install "c:\tools" stable_windows_10_msbuild_Release_Win32_graphviz-2.46.1-win32.zip
-
-python -m pip install --upgrade pip wheel
-
-python -m pip install -r %SMING_HOME%/../docs/requirements.txt
-
-python -m pip uninstall -y xcffib
-
-goto :EOF
-
-:install
-if "%~1"=="" goto :EOF
-mkdir %1
-curl -LO %SMINGTOOLS%/%2
-7z -o%1 -y x %2
+echo.

Sming/Makefile

@@ -103,7 +103,7 @@ docs: submodules ##Build the Sming documentation
 
 # For integration testing both samples and tests are moved outside of the repo.
 SMING_PROJECTS_DIR	?= $(abspath $(SMING_HOME)/..)
-SMING_PROJECTS_DIR	:= $(call FixPath, $(SMING_PROJECTS_DIR))
+SMING_PROJECTS_DIR	:= $(call FixPath,$(SMING_PROJECTS_DIR))
 SAMPLES_DIR			:= $(SMING_PROJECTS_DIR)/samples
 
 # Marks sample as build complete for faster re-testing

Sming/component-wrapper.mk

@@ -153,7 +153,7 @@ endef
 # $1 -> source root directory
 # $2 -> file path(s)
 define ResolveObjPath
-$(foreach f,$2,$(patsubst $(SMING_HOME)/%,%,$(patsubst $1/%,%,$f)))
+$(foreach f,$2,$(patsubst /%,%,$(patsubst $(SMING_HOME)/%,%,$(patsubst $1/%,%,$f))))
 endef
 
 # All source files, absolute paths

Sming/project.mk

@@ -261,7 +261,8 @@ endif
 CONFIG_DEBUG_FILE	:= $(OUT_BASE)/debug.mk
 
 # Append standard search directories to any defined by the application
-ALL_SEARCH_DIRS			:= $(call FixPath,$(abspath $(COMPONENT_SEARCH_DIRS)))
+COMPONENT_SEARCH_DIRS	:= $(call FixPath,$(COMPONENT_SEARCH_DIRS))
+ALL_SEARCH_DIRS			:= $(abspath $(COMPONENT_SEARCH_DIRS))
 COMPONENTS_EXTRA_INCDIR	+= $(ALL_SEARCH_DIRS)
 ALL_SEARCH_DIRS			+= $(ARCH_COMPONENTS) $(SMING_HOME)/Components $(SMING_HOME)/Libraries
 

Sming/util.mk

@@ -9,7 +9,7 @@
 ifeq ($(OS),Windows_NT)
 # Powershell does weird things to this variable, revert to default
 override MAKE	:= make
-FixPath			= $(subst //,/,$(subst \,/,$(addprefix /,$(subst :,,$1))))
+FixPath			= $(subst //,/,$(subst \,/,/$(subst :,,$1)))
 else
 FixPath			= $1
 endif

Tools/ci/build.cmd

@@ -11,7 +11,7 @@ call Arch\%SMING_ARCH%\Tools\ci\build.setup.cmd || goto :error
 
 env
 
-set MAKE_PARALLEL=make -j2
+set MAKE_PARALLEL=make -j%NUMBER_OF_PROCESSORS%
 
 REM Move samples and tests into directory outside of the Sming repo.
 set SMING_PROJECTS_DIR=%CI_BUILD_DIR%\projects
@@ -27,6 +27,14 @@ cd /d %SMING_PROJECTS_DIR%/samples/Basic_Blink
 make help
 make list-config
 
+REM HostTests should build and run on all architectures
+if "%BUILD_COMPILER%" == "udk" ( 
+    REM Skip old toolchain - there are issues
+) else (
+    %MAKE_PARALLEL% -C "%SMING_PROJECTS_DIR%/tests/HostTests"
+)
+
+REM Start Arch-specific tests
 cd /d %SMING_HOME%
 call Arch\%SMING_ARCH%\Tools\ci\build.run.cmd || goto :error
 goto :EOF

Tools/ci/build.sh

@@ -2,7 +2,7 @@
 set -ex # exit with nonzero exit code if anything fails
 
 # Build times benefit from parallel building
-export MAKE_PARALLEL="make -j3"
+export MAKE_PARALLEL="make -j$(nproc)"
 
 cd "$SMING_HOME"
 source "Arch/$SMING_ARCH/Tools/ci/build.setup.sh"

Tools/ci/install.cmd

@@ -1,11 +1,36 @@
-REM Windows install script
-
-set SMINGTOOLS=https://github.com/SmingHub/SmingTools/releases/download/1.0
+REM
+REM Windows CI install script
+REM
 
+echo.
+echo.
+echo ** Installing common python requirements
+echo.
 python -m pip install --upgrade pip -r %SMING_HOME%\..\Tools\requirements.txt
 
+echo.
+echo.
+echo ** Installing MinGW
+echo.
 rmdir /s /q c:\MinGW
 curl -Lo MinGW.7z %SMINGTOOLS%/MinGW-2020-10-19.7z
 7z -oC:\ x MinGW.7z
 
-call %SMING_HOME%\Arch\%SMING_ARCH%\Tools\ci\install.cmd
+if "%1" == "all" (
+    call :install Host Esp8266 Esp32 Rp2040
+    goto :EOF
+)
+
+:install
+if "%1" == "" goto :EOF
+echo.
+echo.
+echo ** Installing %1 toolchain
+echo.
+if "%1" == "doc" (
+    call %SMING_HOME%\..\docs\Tools\install.cmd
+) else (
+    call %SMING_HOME%\Arch\%1\Tools\ci\install.cmd
+)
+shift
+goto :install

Tools/ci/install.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# CI installer script
+#
+# May also be used for library CI builds. The following variables must be set:
+#
+#   CI_BUILD_DIR
+#   SMING_HOME
+#
+
+# appveyor-specific
+export PYTHON=$HOME/venv3.9/bin/python
+export ESP32_PYTHON_PATH=$HOME/venv3.9/bin
+source "$HOME/venv3.9/bin/activate"
+
+# Install requested toolchains
+"$SMING_HOME/../Tools/install.sh" $@

Tools/ci/setenv.ps1

@@ -0,0 +1,26 @@
+
+# Esp8266 
+$env:UDK_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-alt-sdk"
+$env:EQT_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-quick-toolchain"
+if ($env:BUILD_COMPILER -eq "udk") {
+$env:ESP_HOME = $env:UDK_ROOT
+} else {
+$env:ESP_HOME = $env:EQT_ROOT
+}
+
+# Esp32
+$env:IDF_PATH = Join-Path $env:CI_BUILD_DIR "opt/esp-idf"
+$env:IDF_TOOLS_PATH = Join-Path $env:CI_BUILD_DIR "opt/tools/esp32"
+$env:IDF_BRANCH = "sming/dev/v4.3"
+
+# Rp2040
+$env:PICO_TOOLCHAIN_PATH = Join-Path $env:CI_BUILD_DIR "opt/rp2040"
+
+# General
+$env:SMINGTOOLS = "https://github.com/SmingHub/SmingTools/releases/download/1.0"
+
+if ($IsWindows) {
+    $env:PATH = "C:\Python39;C:\Python39\Scripts;C:\MinGW\msys\1.0\bin;C:\MinGW\bin;" + $env:PATH
+    $env:PYTHON = "C:\Python39\python"
+    $env:ESP32_PYTHON_PATH = "C:\Python39"
+}

Tools/install.sh

@@ -11,15 +11,16 @@ inst_host=0
 inst_doc=0
 inst_esp8266=0
 inst_esp32=0
+inst_rp2040=0
 err=0
 
 for opt in "$@"; do
     case $opt in
         all)
             inst_host=1
-            inst_doc=1
             inst_esp8266=1
             inst_esp32=1
+            inst_rp2040=1
             ;;
 
         host | doc | esp8266 | esp32 | rp2040)
@@ -36,11 +37,11 @@ done
 if [[ $err -eq 1 ]] || [ $# -eq 0 ]; then
     echo 'Sming Installation options:'
     echo '   host      Host development tools'
-    echo '   doc       Tools required to build documentation'
     echo '   esp8266   ESP8266 development tools'
     echo '   esp32     ESP32 development tools'
     echo '   rp2040    RP2040 tools (Raspberry Pi Pico)'
-    echo '   all       Install everything'
+    echo '   all       Install all architectures'
+    echo '   doc       Tools required to build documentation'
     echo
     if [ $sourced = 1 ]; then
         return 1
@@ -158,6 +159,10 @@ python3 -m pip install --upgrade pip -r "$SMING_HOME/../Tools/requirements.txt"
 
 
 install() {
+    echo
+    echo
+    echo "** Installing $1 toolchain"
+    echo
     source "$SMING_HOME/Arch/$1/Tools/install.sh"
 }
 

appveyor.yml

@@ -44,36 +44,14 @@ install:
       if ($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME) {
         $env:CI_PULL_REQUEST = "true"
       }
-      # Esp8266 
-      $env:UDK_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-alt-sdk"
-      $env:EQT_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-quick-toolchain"
-      if ($env:BUILD_COMPILER -eq "udk") {
-        $env:ESP_HOME = $env:UDK_ROOT
-      } else {
-        $env:ESP_HOME = $env:EQT_ROOT
-      }
-      # Esp32
-      $env:IDF_PATH = Join-Path $env:CI_BUILD_DIR "opt/esp-idf"
-      $env:IDF_TOOLS_PATH = Join-Path $env:CI_BUILD_DIR "opt/tools/esp32"
-      $env:IDF_BRANCH = "sming/dev/v4.3"
-      # Rp2040
-      $env:PICO_TOOLCHAIN_PATH = Join-Path $env:CI_BUILD_DIR "opt/rp2040"
-      # General
       $env:SMING_HOME = Join-Path $env:CI_BUILD_DIR "Sming"
+      Tools/ci/setenv.ps1
 
   - cmd: |
-      set PATH=C:\Python39;C:\Python39\Scripts;C:\MinGW\msys\1.0\bin;C:\MinGW\bin;%PATH%
-      set PYTHON=C:\Python39\python
-      set ESP32_PYTHON_PATH=C:\Python39
-      Tools\ci\install.cmd
-      set PATH=C:\tools\doxygen;C:\tools\Graphviz\bin;%PATH%
+      Tools\ci\install.cmd %INSTALL_OPTS%
 
   - sh: |
-      export PYTHON=$HOME/venv3.9/bin/python
-      export ESP32_PYTHON_PATH=$HOME/venv3.9/bin
-      source $HOME/venv3.9/bin/activate
-      Tools/install.sh $INSTALL_OPTS
-      cd $CI_BUILD_DIR
+      . Tools/ci/install.sh $INSTALL_OPTS
 
 
 before_build:

docs/Tools/install.cmd

@@ -0,0 +1,17 @@
+REM Docs install.cmd
+
+call :install "c:\tools\doxygen" doxygen-1.9.1.windows.bin.zip
+call :install "c:\tools" stable_windows_10_msbuild_Release_Win32_graphviz-2.46.1-win32.zip
+setx PATH C:\tools\doxygen;C:\tools\Graphviz\bin;%PATH%
+
+python -m pip install --upgrade pip wheel
+python -m pip install -r %SMING_HOME%/../docs/requirements.txt
+python -m pip uninstall -y xcffib
+
+goto :EOF
+
+:install
+if "%~1"=="" goto :EOF
+mkdir %1
+curl -LO %SMINGTOOLS%/%2
+7z -o%1 -y x %2