Fix IDF tool installation (#2861)

This PR fixes tool installation for IDF < 5.2.

Overlooked that wildcard tool spec. only supported for IDF 5.2 installer.
Fixed by providing explicit list of tools for each version.

**Improve cache-clean action**

Not very intuitive so added four levels of cleaning:

- pull-requests: Any items created by pull requests, excludes anything in develop branch
- ccache: All ccache items
- idf-tools: All IDF tools
- ccache+idf: All ccache and IDF tool caches.

Removed the `cache-rebuild` action since we can just do a cache clean then manually trigger a rebuild of the develop branch to accomplish a similar thing.

**Update `clean-tools.py` script**

Didn't clean some tools.

.github/workflows/cache-clean.yml

@@ -2,6 +2,16 @@ name: Cache clean
 
 on:
   workflow_dispatch:
+    inputs:
+      clean_opt:
+        description: 'Level of cleaning required'
+        type: choice
+        default: push-requests
+        options:
+          - pull-requests
+          - ccache
+          - idf-tools
+          - ccache+idf
 
 jobs:
   cleanup:
@@ -12,7 +22,26 @@ jobs:
           gh extension install actions/gh-actions-cache
 
           echo "Fetching list of cache keys"
-          cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep -v develop | cut -f 1 )
+          case $CLEAN_OPT in
+            pull-requests)
+              filter="-v develop"
+              ;;
+            ccache)
+              filter="ccache"
+              ;;
+            idf-tools)
+              filter="idf"
+              ;;
+            ccache+idf)
+              filter="ccache\|idf"
+              ;;
+            *)
+              echo "Unknown option '$CLEAN_OPT'"
+              exit 1
+              ;;
+          esac
+
+          cacheKeys=$(gh actions-cache list -R $REPO -L 100 | grep $filter | cut -f 1 )
 
           echo "Deleting caches..."
           set +e
@@ -23,3 +52,4 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
+          CLEAN_OPT: ${{ inputs.clean_opt }}

Sming/Arch/Esp32/Tools/idf_tools-4.3.lst

@@ -0,0 +1,7 @@
+xtensa-esp32-elf
+xtensa-esp32s2-elf
+xtensa-esp32s3-elf
+riscv32-esp-elf
+esp32ulp-elf
+esp32s2ulp-elf
+openocd-esp32

Sming/Arch/Esp32/Tools/idf_tools-4.4.lst

@@ -0,0 +1,8 @@
+xtensa-esp32-elf
+xtensa-esp32s2-elf
+xtensa-esp32s3-elf
+riscv32-esp-elf
+esp32ulp-elf
+xtensa-esp-elf-gdb
+riscv32-esp-elf-gdb
+openocd-esp32

Sming/Arch/Esp32/Tools/idf_tools-5.0.lst

@@ -0,0 +1,9 @@
+xtensa-esp-elf-gdb
+riscv32-esp-elf-gdb
+xtensa-esp32-elf
+xtensa-esp32s2-elf
+xtensa-esp32s3-elf
+riscv32-esp-elf
+esp32ulp-elf
+esp-rom-elfs
+openocd-esp32

Sming/Arch/Esp32/Tools/idf_tools-5.2.lst

@@ -0,0 +1,7 @@
+xtensa-esp-elf-gdb
+riscv32-esp-elf-gdb
+xtensa-esp-elf
+riscv32-esp-elf
+esp32ulp-elf
+esp-rom-elfs
+openocd-esp32

Sming/Arch/Esp32/Tools/install.cmd

@@ -45,18 +45,21 @@ goto :setup
 if exist "%IDF_PATH%" rmdir /q %IDF_PATH%
 mklink /j %IDF_PATH% %IDF_CLONE_PATH%
 
-if "%CI_BUILD_DIR%"=="" (
-set IDF_TOOL_PACKAGES=*openocd*
-) else (
-    REM Skip installation for CI if already present
+REM Install IDF tools and packages (unless CI has restored from cache)
+if "%CI_BUILD_DIR%" NEQ "" (
     if exist "%IDF_TOOLS_PATH%\tools" (
+        echo Skipping IDF tools installation, "%IDF_TOOLS_PATH%\tools" exists
         goto :EOF
     )
 )
 
-REM Install IDF tools and packages
-python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install install "*elf*" %IDF_TOOL_PACKAGES%
-python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env
+REM Be specific about which tools we want to install for each IDF version
+tr '\n' ' ' < "%~dp0idf_tools-%INSTALL_IDF_VER%.lst" > toolver.txt || goto :EOF
+for /f "tokens=*" %%x in (toolver.txt) do set IDF_TOOL_PACKAGES=%%x
+del toolver.txt
+echo Install: %IDF_TOOL_PACKAGES%
+python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install %IDF_TOOL_PACKAGES% || goto :EOF
+python "%IDF_PATH%\tools\idf_tools.py" --non-interactive install-python-env || goto :EOF
 
 if "%CI_BUILD_DIR%" NEQ "" (
     del /q "%IDF_TOOLS_PATH%\dist\*"

Sming/Arch/Esp32/Tools/install.sh

@@ -61,22 +61,24 @@ rm -f "$IDF_PATH"
 ln -s "$IDF_CLONE_PATH" "$IDF_PATH"
 
 
-# Skip installation for CI if already present
-if [ -z "$CI_BUILD_DIR" ] || [ ! -d "$IDF_TOOLS_PATH/tools" ]; then
-
-# Install IDF tools and packages
-python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install "*elf*"
-if [ -n "$VIRTUAL_ENV" ]; then
-    unset VIRTUAL_ENV
-    unset VIRTUAL_ENV_PROMPT
-    export PATH="${PATH/$VIRTUAL_ENV\/bin:/}"
-fi
-python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install-python-env
+# Install IDF tools and packages (unless CI has restored from cache)
+if [ -n "$CI_BUILD_DIR" ] && [ -d "$IDF_TOOLS_PATH/tools" ]; then
+    printf "\n\n** Skipping IDF tools installation: '%s/tools' exists\n\n" "$IDF_TOOLS_PATH"
+else
+    # Be specific about which tools we want to install for each IDF version
+    IDF_TOOL_PACKAGES=$(tr '\n' ' ' < "$SMING_HOME/Arch/Esp32/Tools/idf_tools-${INSTALL_IDF_VER}.lst")
+    echo "Install: $IDF_TOOL_PACKAGES"
+    python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install $IDF_TOOL_PACKAGES
+    if [ -n "$VIRTUAL_ENV" ]; then
+        unset VIRTUAL_ENV
+        unset VIRTUAL_ENV_PROMPT
+        export PATH="${PATH/$VIRTUAL_ENV\/bin:/}"
+    fi
+    python3 "$IDF_PATH/tools/idf_tools.py" --non-interactive install-python-env
 
-if [ -z "$KEEP_DOWNLOADS" ]; then
-    rm -rf "$IDF_TOOLS_PATH/dist"
+    if [ -z "$KEEP_DOWNLOADS" ]; then
+        rm -rf "$IDF_TOOLS_PATH/dist"
+    fi
 fi
 
-fi # CI install
-
 fi

Tools/ci/README.rst

@@ -7,11 +7,31 @@ Github actions
 See ``.github/workflows``.
 
 Cache clean
-      Dispatch workflow to remove caches for pull requests, but leave those for the default (develop) branch intact.
+      Dispatch workflow as convenient way to clean action cache. Cleaning levels are:
+
+      - pull-requests
+            Any items created by pull requests, excludes anything in develop branch.
+            Normally pull requests will make use of caches present in the develop branch, but if there isn't one it will create its own. This becomes redundant when merged to develop.
+
+            Such caches should be expired automatically, but this option can be used to remove them.
+
+      - ccache
+            All ccache items. Use if pull requests are taking too long to build.
+
+      - idf-tools
+            All IDF tool. Use before merging to develop if IDF toolchains have been updated.
+
+      - ccache+idf
+            All ccache and IDF tool caches.
+
+      Note that cleaning can always be done locally using ``gh``::
+
+            gh cache list # For fork
+            gh cache list -R SmingHub/Sming
+            gh cache delete --all -R SmingHub/Sming
+
+      etc.
 
-Cache rebuild
-      Dispatch workflow to rebuild the esp32 idf/tool caches.
-      By default, cleans only idf/esp32 caches but has option to perform full clean.
 
 CodeQL
       Performs code quality analysis when develop branch is updated.

Tools/ci/clean-tools.py

@@ -20,10 +20,9 @@
 # These filters are matched from the **start** of the path so there's an implicit .* at the end
 FILTERS = {
     IDF_TOOLS_PATH: [
-        # Leave versioned directory to avoid re-installation
-        r'.*esp-elf-gdb/.*/.*esp-elf-gdb/',
-        r'esp32ulp-elf/.*/esp32ulp-elf/',
-        r'openocd-esp32/.*/openocd-esp32/',
+        r'tools/.*esp-elf-gdb/',
+        r'tools/esp32ulp-elf/',
+        r'tools/openocd-esp32/',
         # Libraries not required by Sming
         # r'.*/riscv32-esp-elf/lib/{ARC}',
         r'.*/riscv32-esp-elf/lib/rv32i_.*',