From 0ddd8317d793d5b310fb921aeb5754b8ddc9c3b7 Mon Sep 17 00:00:00 2001 From: Mike Date: Fri, 19 Nov 2021 08:37:07 +0000 Subject: [PATCH] Build and installer fixes (#2429) **Host** - Fix host --help option - Add --loopcount emulator option **Build** - Update FixPath to handle spaces in path - Fix COMPONENT_SEARCH_DIRS for Windows - Fix appcode object path resolution, ensure object path is relative **Installer** - `all` option missing rp2040 - Remove `doc` option from `all` **CI testing** - Revise to simplify use by external libraries Co-authored-by: Slavey Karadzhov --- Sming/Arch/Host/Components/hostlib/options.h | 6 ++-- .../Arch/Host/Components/hostlib/startup.cpp | 13 ++++++++ Sming/Arch/Host/Tools/ci/install.cmd | 17 +--------- Sming/Makefile | 2 +- Sming/component-wrapper.mk | 2 +- Sming/project.mk | 3 +- Sming/util.mk | 2 +- Tools/ci/build.cmd | 10 +++++- Tools/ci/build.sh | 2 +- Tools/ci/install.cmd | 33 ++++++++++++++++--- Tools/ci/install.sh | 17 ++++++++++ Tools/ci/setenv.ps1 | 26 +++++++++++++++ Tools/install.sh | 11 +++++-- appveyor.yml | 28 ++-------------- docs/Tools/install.cmd | 17 ++++++++++ 15 files changed, 133 insertions(+), 56 deletions(-) create mode 100755 Tools/ci/install.sh create mode 100644 Tools/ci/setenv.ps1 create mode 100644 docs/Tools/install.cmd diff --git a/Sming/Arch/Host/Components/hostlib/options.h b/Sming/Arch/Host/Components/hostlib/options.h index 9634e0b2fa..45e7603d4e 100644 --- a/Sming/Arch/Host/Components/hostlib/options.h +++ b/Sming/Arch/Host/Components/hostlib/options.h @@ -32,9 +32,9 @@ XX(uart, required_argument, "Enable UART server", "PORT", "Which UART number to enable", \ "e.g. --uart=0 --uart=1 enable servers for UART0, UART1\0") \ XX(device, required_argument, "Set device for uart", "DEVICE", "Optionally map uart to device", \ - "e.g. --uart=0 --device=/dev/ttyUSB0") \ + "e.g. --uart=0 --device=/dev/ttyUSB0\0") \ XX(baud, required_argument, "Set baud rate for UART", "BAUD", "Requires --device argument", \ - "e.g. --uart=0 --device=/dev/ttyUSB0 --baud=115200") \ + "e.g. --uart=0 --device=/dev/ttyUSB0 --baud=115200\0") \ XX(portbase, required_argument, "Specify base port number for UART socket servers", "PORT", "IP port number", \ nullptr) \ XX(ifname, required_argument, "Specify network interface", "NAME", "Network interface to use (e.g. tap0)", \ @@ -52,6 +52,8 @@ XX(flashsize, required_argument, "Change default flash size if file doesn't exist", "SIZE", \ "Size of flash in bytes (e.g. 512K, 524288, 0x80000)", nullptr) \ XX(initonly, no_argument, "Initialise only, do not start Sming", nullptr, nullptr, nullptr) \ + XX(loopcount, required_argument, "Run Sming loop a fixed number of times then exit", nullptr, nullptr, \ + "Useful for running samples in CI\0") \ XX(nonet, no_argument, "Skip network initialisation", nullptr, nullptr, nullptr) \ XX(debug, required_argument, "Set debug verbosity", "LEVEL", "Maximum debug message level to print", \ "0 = errors only, 1 = +warnings, 2 = +info\0") diff --git a/Sming/Arch/Host/Components/hostlib/startup.cpp b/Sming/Arch/Host/Components/hostlib/startup.cpp index 71ccab74f4..b36e89f342 100644 --- a/Sming/Arch/Host/Components/hostlib/startup.cpp +++ b/Sming/Arch/Host/Components/hostlib/startup.cpp @@ -134,6 +134,7 @@ int main(int argc, char* argv[]) int pause; int exitpause; bool initonly; + int loopcount; bool enable_network; UartServer::Config uart; FlashmemConfig flash; @@ -244,6 +245,10 @@ int main(int argc, char* argv[]) config.initonly = true; break; + case opt_loopcount: + config.loopcount = atoi(arg); + break; + case opt_nonet: config.enable_network = false; break; @@ -309,6 +314,14 @@ int main(int argc, char* argv[]) #endif while(!done) { host_main_loop(); + if(config.loopcount == 0) { + continue; + } + --config.loopcount; + if(config.loopcount == 0) { + host_debug_i("Reached requested loop count limit: exiting"); + break; + } } host_debug_i(">> Normal Exit <<\n"); diff --git a/Sming/Arch/Host/Tools/ci/install.cmd b/Sming/Arch/Host/Tools/ci/install.cmd index b1218913be..1972448930 100644 --- a/Sming/Arch/Host/Tools/ci/install.cmd +++ b/Sming/Arch/Host/Tools/ci/install.cmd @@ -1,18 +1,3 @@ REM Host install.cmd -call :install "c:\tools\doxygen" doxygen-1.9.1.windows.bin.zip -call :install "c:\tools" stable_windows_10_msbuild_Release_Win32_graphviz-2.46.1-win32.zip - -python -m pip install --upgrade pip wheel - -python -m pip install -r %SMING_HOME%/../docs/requirements.txt - -python -m pip uninstall -y xcffib - -goto :EOF - -:install -if "%~1"=="" goto :EOF -mkdir %1 -curl -LO %SMINGTOOLS%/%2 -7z -o%1 -y x %2 +echo. diff --git a/Sming/Makefile b/Sming/Makefile index a82c0aa6f5..155e9b13df 100644 --- a/Sming/Makefile +++ b/Sming/Makefile @@ -103,7 +103,7 @@ docs: submodules ##Build the Sming documentation # For integration testing both samples and tests are moved outside of the repo. SMING_PROJECTS_DIR ?= $(abspath $(SMING_HOME)/..) -SMING_PROJECTS_DIR := $(call FixPath, $(SMING_PROJECTS_DIR)) +SMING_PROJECTS_DIR := $(call FixPath,$(SMING_PROJECTS_DIR)) SAMPLES_DIR := $(SMING_PROJECTS_DIR)/samples # Marks sample as build complete for faster re-testing diff --git a/Sming/component-wrapper.mk b/Sming/component-wrapper.mk index 06079ebab3..f6da0f6a34 100644 --- a/Sming/component-wrapper.mk +++ b/Sming/component-wrapper.mk @@ -153,7 +153,7 @@ endef # $1 -> source root directory # $2 -> file path(s) define ResolveObjPath -$(foreach f,$2,$(patsubst $(SMING_HOME)/%,%,$(patsubst $1/%,%,$f))) +$(foreach f,$2,$(patsubst /%,%,$(patsubst $(SMING_HOME)/%,%,$(patsubst $1/%,%,$f)))) endef # All source files, absolute paths diff --git a/Sming/project.mk b/Sming/project.mk index c860dbd4c9..75942bc533 100644 --- a/Sming/project.mk +++ b/Sming/project.mk @@ -261,7 +261,8 @@ endif CONFIG_DEBUG_FILE := $(OUT_BASE)/debug.mk # Append standard search directories to any defined by the application -ALL_SEARCH_DIRS := $(call FixPath,$(abspath $(COMPONENT_SEARCH_DIRS))) +COMPONENT_SEARCH_DIRS := $(call FixPath,$(COMPONENT_SEARCH_DIRS)) +ALL_SEARCH_DIRS := $(abspath $(COMPONENT_SEARCH_DIRS)) COMPONENTS_EXTRA_INCDIR += $(ALL_SEARCH_DIRS) ALL_SEARCH_DIRS += $(ARCH_COMPONENTS) $(SMING_HOME)/Components $(SMING_HOME)/Libraries diff --git a/Sming/util.mk b/Sming/util.mk index 5ddcd8ac16..e157049350 100644 --- a/Sming/util.mk +++ b/Sming/util.mk @@ -9,7 +9,7 @@ ifeq ($(OS),Windows_NT) # Powershell does weird things to this variable, revert to default override MAKE := make -FixPath = $(subst //,/,$(subst \,/,$(addprefix /,$(subst :,,$1)))) +FixPath = $(subst //,/,$(subst \,/,/$(subst :,,$1))) else FixPath = $1 endif diff --git a/Tools/ci/build.cmd b/Tools/ci/build.cmd index 9b53440f51..df0e5be3a2 100644 --- a/Tools/ci/build.cmd +++ b/Tools/ci/build.cmd @@ -11,7 +11,7 @@ call Arch\%SMING_ARCH%\Tools\ci\build.setup.cmd || goto :error env -set MAKE_PARALLEL=make -j2 +set MAKE_PARALLEL=make -j%NUMBER_OF_PROCESSORS% REM Move samples and tests into directory outside of the Sming repo. set SMING_PROJECTS_DIR=%CI_BUILD_DIR%\projects @@ -27,6 +27,14 @@ cd /d %SMING_PROJECTS_DIR%/samples/Basic_Blink make help make list-config +REM HostTests should build and run on all architectures +if "%BUILD_COMPILER%" == "udk" ( + REM Skip old toolchain - there are issues +) else ( + %MAKE_PARALLEL% -C "%SMING_PROJECTS_DIR%/tests/HostTests" +) + +REM Start Arch-specific tests cd /d %SMING_HOME% call Arch\%SMING_ARCH%\Tools\ci\build.run.cmd || goto :error goto :EOF diff --git a/Tools/ci/build.sh b/Tools/ci/build.sh index cccaef29dc..10b1047958 100755 --- a/Tools/ci/build.sh +++ b/Tools/ci/build.sh @@ -2,7 +2,7 @@ set -ex # exit with nonzero exit code if anything fails # Build times benefit from parallel building -export MAKE_PARALLEL="make -j3" +export MAKE_PARALLEL="make -j$(nproc)" cd "$SMING_HOME" source "Arch/$SMING_ARCH/Tools/ci/build.setup.sh" diff --git a/Tools/ci/install.cmd b/Tools/ci/install.cmd index aa49e52a34..6430701b7f 100644 --- a/Tools/ci/install.cmd +++ b/Tools/ci/install.cmd @@ -1,11 +1,36 @@ -REM Windows install script - -set SMINGTOOLS=https://github.com/SmingHub/SmingTools/releases/download/1.0 +REM +REM Windows CI install script +REM +echo. +echo. +echo ** Installing common python requirements +echo. python -m pip install --upgrade pip -r %SMING_HOME%\..\Tools\requirements.txt +echo. +echo. +echo ** Installing MinGW +echo. rmdir /s /q c:\MinGW curl -Lo MinGW.7z %SMINGTOOLS%/MinGW-2020-10-19.7z 7z -oC:\ x MinGW.7z -call %SMING_HOME%\Arch\%SMING_ARCH%\Tools\ci\install.cmd +if "%1" == "all" ( + call :install Host Esp8266 Esp32 Rp2040 + goto :EOF +) + +:install +if "%1" == "" goto :EOF +echo. +echo. +echo ** Installing %1 toolchain +echo. +if "%1" == "doc" ( + call %SMING_HOME%\..\docs\Tools\install.cmd +) else ( + call %SMING_HOME%\Arch\%1\Tools\ci\install.cmd +) +shift +goto :install diff --git a/Tools/ci/install.sh b/Tools/ci/install.sh new file mode 100755 index 0000000000..9fc8bc430e --- /dev/null +++ b/Tools/ci/install.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# +# CI installer script +# +# May also be used for library CI builds. The following variables must be set: +# +# CI_BUILD_DIR +# SMING_HOME +# + +# appveyor-specific +export PYTHON=$HOME/venv3.9/bin/python +export ESP32_PYTHON_PATH=$HOME/venv3.9/bin +source "$HOME/venv3.9/bin/activate" + +# Install requested toolchains +"$SMING_HOME/../Tools/install.sh" $@ diff --git a/Tools/ci/setenv.ps1 b/Tools/ci/setenv.ps1 new file mode 100644 index 0000000000..2078001441 --- /dev/null +++ b/Tools/ci/setenv.ps1 @@ -0,0 +1,26 @@ + +# Esp8266 +$env:UDK_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-alt-sdk" +$env:EQT_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-quick-toolchain" +if ($env:BUILD_COMPILER -eq "udk") { +$env:ESP_HOME = $env:UDK_ROOT +} else { +$env:ESP_HOME = $env:EQT_ROOT +} + +# Esp32 +$env:IDF_PATH = Join-Path $env:CI_BUILD_DIR "opt/esp-idf" +$env:IDF_TOOLS_PATH = Join-Path $env:CI_BUILD_DIR "opt/tools/esp32" +$env:IDF_BRANCH = "sming/dev/v4.3" + +# Rp2040 +$env:PICO_TOOLCHAIN_PATH = Join-Path $env:CI_BUILD_DIR "opt/rp2040" + +# General +$env:SMINGTOOLS = "https://github.com/SmingHub/SmingTools/releases/download/1.0" + +if ($IsWindows) { + $env:PATH = "C:\Python39;C:\Python39\Scripts;C:\MinGW\msys\1.0\bin;C:\MinGW\bin;" + $env:PATH + $env:PYTHON = "C:\Python39\python" + $env:ESP32_PYTHON_PATH = "C:\Python39" +} diff --git a/Tools/install.sh b/Tools/install.sh index d0dbe15586..9795528879 100755 --- a/Tools/install.sh +++ b/Tools/install.sh @@ -11,15 +11,16 @@ inst_host=0 inst_doc=0 inst_esp8266=0 inst_esp32=0 +inst_rp2040=0 err=0 for opt in "$@"; do case $opt in all) inst_host=1 - inst_doc=1 inst_esp8266=1 inst_esp32=1 + inst_rp2040=1 ;; host | doc | esp8266 | esp32 | rp2040) @@ -36,11 +37,11 @@ done if [[ $err -eq 1 ]] || [ $# -eq 0 ]; then echo 'Sming Installation options:' echo ' host Host development tools' - echo ' doc Tools required to build documentation' echo ' esp8266 ESP8266 development tools' echo ' esp32 ESP32 development tools' echo ' rp2040 RP2040 tools (Raspberry Pi Pico)' - echo ' all Install everything' + echo ' all Install all architectures' + echo ' doc Tools required to build documentation' echo if [ $sourced = 1 ]; then return 1 @@ -158,6 +159,10 @@ python3 -m pip install --upgrade pip -r "$SMING_HOME/../Tools/requirements.txt" install() { + echo + echo + echo "** Installing $1 toolchain" + echo source "$SMING_HOME/Arch/$1/Tools/install.sh" } diff --git a/appveyor.yml b/appveyor.yml index 35c85c842f..cc063a50ad 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,36 +44,14 @@ install: if ($env:APPVEYOR_PULL_REQUEST_HEAD_REPO_NAME) { $env:CI_PULL_REQUEST = "true" } - # Esp8266 - $env:UDK_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-alt-sdk" - $env:EQT_ROOT = Join-Path $env:CI_BUILD_DIR "opt/esp-quick-toolchain" - if ($env:BUILD_COMPILER -eq "udk") { - $env:ESP_HOME = $env:UDK_ROOT - } else { - $env:ESP_HOME = $env:EQT_ROOT - } - # Esp32 - $env:IDF_PATH = Join-Path $env:CI_BUILD_DIR "opt/esp-idf" - $env:IDF_TOOLS_PATH = Join-Path $env:CI_BUILD_DIR "opt/tools/esp32" - $env:IDF_BRANCH = "sming/dev/v4.3" - # Rp2040 - $env:PICO_TOOLCHAIN_PATH = Join-Path $env:CI_BUILD_DIR "opt/rp2040" - # General $env:SMING_HOME = Join-Path $env:CI_BUILD_DIR "Sming" + Tools/ci/setenv.ps1 - cmd: | - set PATH=C:\Python39;C:\Python39\Scripts;C:\MinGW\msys\1.0\bin;C:\MinGW\bin;%PATH% - set PYTHON=C:\Python39\python - set ESP32_PYTHON_PATH=C:\Python39 - Tools\ci\install.cmd - set PATH=C:\tools\doxygen;C:\tools\Graphviz\bin;%PATH% + Tools\ci\install.cmd %INSTALL_OPTS% - sh: | - export PYTHON=$HOME/venv3.9/bin/python - export ESP32_PYTHON_PATH=$HOME/venv3.9/bin - source $HOME/venv3.9/bin/activate - Tools/install.sh $INSTALL_OPTS - cd $CI_BUILD_DIR + . Tools/ci/install.sh $INSTALL_OPTS before_build: diff --git a/docs/Tools/install.cmd b/docs/Tools/install.cmd new file mode 100644 index 0000000000..352b68edd1 --- /dev/null +++ b/docs/Tools/install.cmd @@ -0,0 +1,17 @@ +REM Docs install.cmd + +call :install "c:\tools\doxygen" doxygen-1.9.1.windows.bin.zip +call :install "c:\tools" stable_windows_10_msbuild_Release_Win32_graphviz-2.46.1-win32.zip +setx PATH C:\tools\doxygen;C:\tools\Graphviz\bin;%PATH% + +python -m pip install --upgrade pip wheel +python -m pip install -r %SMING_HOME%/../docs/requirements.txt +python -m pip uninstall -y xcffib + +goto :EOF + +:install +if "%~1"=="" goto :EOF +mkdir %1 +curl -LO %SMINGTOOLS%/%2 +7z -o%1 -y x %2