From caa391ab3420db0316b8ad74ba13d8fc3ac66a9e Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Mon, 30 Sep 2019 12:10:48 +0300 Subject: [PATCH] More efficient CI builds (#3303) * More efficient CI builds * Update main.yml --- .github/workflows/main.yml | 33 ++++++--------- tools/ci/install-arduino-core-esp32.sh | 7 ++-- tools/ci/install-arduino-ide.sh | 42 ++++++++++++------- tools/ci/install-platformio-esp32.sh | 31 +++++++++++--- tools/ci/on-push.sh | 56 ++++++++++++++++---------- 5 files changed, 103 insertions(+), 66 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b56dcbf9761..dfd624710b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,43 +8,34 @@ on: pull_request: jobs: - + + # Ubuntu build-arduino-linux: name: Arduino ${{ matrix.chunk }} on ubuntu-latest runs-on: ubuntu-latest strategy: matrix: - chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11] + chunk: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14] steps: - uses: actions/checkout@v1 - name: Build Sketches - run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 12 + run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 15 - build-arduino-mac: - name: Arduino ${{ matrix.chunk }} on macOS-latest - runs-on: macOS-latest - strategy: - matrix: - chunk: [3, 6] - - steps: - - uses: actions/checkout@v1 - - name: Build Sketches - run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 12 - - build-arduino-win: - name: Arduino ${{ matrix.chunk }} on windows-latest - runs-on: windows-latest + # Windows and MacOS + build-arduino-win-mac: + name: Arduino on ${{ matrix.os }} + runs-on: ${{ matrix.os }} strategy: matrix: - chunk: [5, 15, 25] + os: [windows-latest, macOS-latest] steps: - uses: actions/checkout@v1 - name: Build Sketches - run: bash ./tools/ci/on-push.sh ${{ matrix.chunk }} 30 + run: bash ./tools/ci/on-push.sh + # PlatformIO on Windows, Ubuntu and Mac build-platformio: name: PlatformIO on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -55,4 +46,4 @@ jobs: steps: - uses: actions/checkout@v1 - name: Build Sketches - run: bash ./tools/ci/on-push.sh 1 1 + run: bash ./tools/ci/on-push.sh 1 1 #equal and non-zero to trigger PIO diff --git a/tools/ci/install-arduino-core-esp32.sh b/tools/ci/install-arduino-core-esp32.sh index fb90e3bf194..271ce2883a2 100644 --- a/tools/ci/install-arduino-core-esp32.sh +++ b/tools/ci/install-arduino-core-esp32.sh @@ -1,7 +1,8 @@ #!/bin/bash -if [ ! -d "$ARDUINO_USR_PATH/hardware/espressif/esp32" ]; then - echo "Installing ESP32 Arduino Core in '$ARDUINO_USR_PATH/hardware/espressif/esp32'..." +export ARDUINO_ESP32_PATH="$ARDUINO_USR_PATH/hardware/espressif/esp32" +if [ ! -d "$ARDUINO_ESP32_PATH" ]; then + echo "Installing ESP32 Arduino Core in '$ARDUINO_ESP32_PATH'..." script_init_path="$PWD" mkdir -p "$ARDUINO_USR_PATH/hardware/espressif" && \ cd "$ARDUINO_USR_PATH/hardware/espressif" @@ -36,6 +37,6 @@ if [ ! -d "$ARDUINO_USR_PATH/hardware/espressif/esp32" ]; then if [ $? -ne 0 ]; then echo "ERROR: Download failed"; exit 1; fi cd $script_init_path - echo "ESP32 Arduino has been installed in '$ARDUINO_USR_PATH/hardware/espressif/esp32'" + echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'" echo "" fi diff --git a/tools/ci/install-arduino-ide.sh b/tools/ci/install-arduino-ide.sh index 05e10edfac8..1a5e3d46a53 100644 --- a/tools/ci/install-arduino-ide.sh +++ b/tools/ci/install-arduino-ide.sh @@ -45,9 +45,8 @@ else export ARDUINO_USR_PATH="$HOME/Arduino" fi -echo "Installing Arduino IDE on $OS_NAME..." - if [ ! -d "$ARDUINO_IDE_PATH" ]; then + echo "Installing Arduino IDE on $OS_NAME..." echo "Downloading 'arduino-nightly-$OS_NAME.$ARCHIVE_FORMAT' to 'arduino.$ARCHIVE_FORMAT'..." if [ "$OS_IS_LINUX" == "1" ]; then wget -O "arduino.$ARCHIVE_FORMAT" "https://www.arduino.cc/download.php?f=/arduino-nightly-$OS_NAME.$ARCHIVE_FORMAT" > /dev/null 2>&1 @@ -70,12 +69,21 @@ if [ ! -d "$ARDUINO_IDE_PATH" ]; then fi if [ $? -ne 0 ]; then exit 1; fi rm -rf "arduino.$ARCHIVE_FORMAT" -fi -mkdir -p "$ARDUINO_USR_PATH/libraries" -mkdir -p "$ARDUINO_USR_PATH/hardware" + mkdir -p "$ARDUINO_USR_PATH/libraries" + mkdir -p "$ARDUINO_USR_PATH/hardware" + + echo "Arduino IDE Installed in '$ARDUINO_IDE_PATH'" + echo "" +fi function build_sketch(){ # build_sketch [extra-options] + if [ "$#" -lt 2 ]; then + echo "ERROR: Illegal number of parameters" + echo "USAGE: build_sketch [extra-options]" + return 1 + fi + local fqbn="$1" local sketch="$2" local xtra_opts="$3" @@ -126,14 +134,26 @@ function count_sketches() # count_sketches return $sketchnum } -function build_sketches() # build_sketches [extra-options] +function build_sketches() # build_sketches [extra-options] { - local examples=$1 - local fqbn=$2 + local fqbn=$1 + local examples=$2 local chunk_idex=$3 local chunks_num=$4 local xtra_opts=$5 + if [ "$#" -lt 2 ]; then + echo "ERROR: Illegal number of parameters" + echo "USAGE: build_sketches [ ] [extra-options]" + return 1 + fi + + if [ "$#" -lt 4 ]; then + chunk_idex="0" + chunks_num="1" + xtra_opts=$3 + fi + if [ "$chunks_num" -le 0 ]; then echo "ERROR: Chunks count must be positive number" return 1 @@ -195,9 +215,3 @@ function build_sketches() # build_sketches [extra-options]' to build your sketches" -echo "" - diff --git a/tools/ci/install-platformio-esp32.sh b/tools/ci/install-platformio-esp32.sh index 503e7d0095e..e822a5ca358 100644 --- a/tools/ci/install-platformio-esp32.sh +++ b/tools/ci/install-platformio-esp32.sh @@ -1,5 +1,7 @@ #!/bin/bash +export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32" + echo "Installing Python Wheel..." pip install wheel > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "ERROR: Install failed"; exit 1; fi @@ -23,10 +25,10 @@ if [ $? -ne 0 ]; then echo "ERROR: Replace failed"; exit 1; fi if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ]; then echo "Linking Core..." && \ - ln -s $GITHUB_WORKSPACE "$HOME/.platformio/packages/framework-arduinoespressif32" + ln -s $GITHUB_WORKSPACE "$PLATFORMIO_ESP32_PATH" else echo "Cloning Core Repository..." && \ - git clone https://github.com/espressif/arduino-esp32.git "$HOME/.platformio/packages/framework-arduinoespressif32" > /dev/null 2>&1 + git clone https://github.com/espressif/arduino-esp32.git "$PLATFORMIO_ESP32_PATH" > /dev/null 2>&1 if [ $? -ne 0 ]; then echo "ERROR: GIT clone failed"; exit 1; fi fi @@ -34,7 +36,13 @@ echo "PlatformIO for ESP32 has been installed" echo "" -function build_pio_sketch(){ # build_pio_sketch [extra-options] +function build_pio_sketch(){ # build_pio_sketch + if [ "$#" -lt 2 ]; then + echo "ERROR: Illegal number of parameters" + echo "USAGE: build_pio_sketch " + return 1 + fi + local board="$1" local sketch="$2" local sketch_dir=$(dirname "$sketch") @@ -65,13 +73,24 @@ function count_sketches() # count_sketches return $sketchnum } -function build_pio_sketches() # build_pio_sketches +function build_pio_sketches() # build_pio_sketches { - local examples=$1 - local board=$2 + if [ "$#" -lt 2 ]; then + echo "ERROR: Illegal number of parameters" + echo "USAGE: build_pio_sketches [ ]" + return 1 + fi + + local board=$1 + local examples=$2 local chunk_idex=$3 local chunks_num=$4 + if [ "$#" -lt 4 ]; then + chunk_idex="0" + chunks_num="1" + fi + if [ "$chunks_num" -le 0 ]; then echo "ERROR: Chunks count must be positive number" return 1 diff --git a/tools/ci/on-push.sh b/tools/ci/on-push.sh index 7fbb477e67f..54df1da88f4 100755 --- a/tools/ci/on-push.sh +++ b/tools/ci/on-push.sh @@ -19,38 +19,50 @@ fi CHUNK_INDEX=$1 CHUNKS_CNT=$2 BUILD_PIO=0 -if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then - echo "Building all sketches" +if [ "$#" -lt 2 ] || [ "$CHUNKS_CNT" -le 0 ]; then CHUNK_INDEX=0 CHUNKS_CNT=1 -fi -if [ "$CHUNK_INDEX" -gt "$CHUNKS_CNT" ]; then +elif [ "$CHUNK_INDEX" -gt "$CHUNKS_CNT" ]; then CHUNK_INDEX=$CHUNKS_CNT -fi -if [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then +elif [ "$CHUNK_INDEX" -eq "$CHUNKS_CNT" ]; then BUILD_PIO=1 fi -# CMake Test -if [ "$CHUNK_INDEX" -eq 0 ]; then - bash ./tools/ci/check-cmakelists.sh - if [ $? -ne 0 ]; then exit 1; fi -fi - if [ "$BUILD_PIO" -eq 0 ]; then # ArduinoIDE Test + FQBN="espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app" source ./tools/ci/install-arduino-ide.sh source ./tools/ci/install-arduino-core-esp32.sh - build_sketches "$GITHUB_WORKSPACE/libraries" "espressif:esp32:esp32:PSRAM=enabled,PartitionScheme=huge_app" "$CHUNK_INDEX" "$CHUNKS_CNT" + if [ "$OS_IS_WINDOWS" == "1" ]; then + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" + elif [ "$OS_IS_MACOS" == "1" ]; then + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \ + build_sketch "$FQBN" "$ARDUINO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" + else + # CMake Test + if [ "$CHUNK_INDEX" -eq 0 ]; then + bash "$ARDUINO_ESP32_PATH/tools/ci/check-cmakelists.sh" + if [ $? -ne 0 ]; then exit 1; fi + fi + build_sketches "$FQBN" "$ARDUINO_ESP32_PATH/libraries" "$CHUNK_INDEX" "$CHUNKS_CNT" + fi else # PlatformIO Test source ./tools/ci/install-platformio-esp32.sh - python -m platformio ci --board esp32dev libraries/WiFi/examples/WiFiClient && \ - python -m platformio ci --board esp32dev libraries/WiFiClientSecure/examples/WiFiClientSecure && \ - python -m platformio ci --board esp32dev libraries/BluetoothSerial/examples/SerialToSerialBT && \ - python -m platformio ci --board esp32dev libraries/BLE/examples/BLE_server && \ - python -m platformio ci --board esp32dev libraries/AzureIoT/examples/GetStarted && \ - python -m platformio ci --board esp32dev libraries/ESP32/examples/Camera/CameraWebServer --project-option="board_build.partitions = huge_app.csv" - #build_pio_sketches libraries esp32dev $CHUNK_INDEX $CHUNKS_CNT - if [ $? -ne 0 ]; then exit 1; fi -fi \ No newline at end of file + BOARD="esp32dev" + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFi/examples/WiFiClient/WiFiClient.ino" && \ + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/WiFiClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino" && \ + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BluetoothSerial/examples/SerialToSerialBT/SerialToSerialBT.ino" && \ + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/BLE/examples/BLE_server/BLE_server.ino" && \ + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/AzureIoT/examples/GetStarted/GetStarted.ino" && \ + build_pio_sketch "$BOARD" "$PLATFORMIO_ESP32_PATH/libraries/ESP32/examples/Camera/CameraWebServer/CameraWebServer.ino" + #build_pio_sketches esp32dev "$PLATFORMIO_ESP32_PATH/libraries" +fi +if [ $? -ne 0 ]; then exit 1; fi