diff --git a/.github/pytools/Sign-File.ps1 b/.github/pytools/Sign-File.ps1 new file mode 100755 index 00000000000..09094096ac7 --- /dev/null +++ b/.github/pytools/Sign-File.ps1 @@ -0,0 +1,78 @@ +[CmdletBinding()] +param ( + [Parameter()] + [String] + $Path +) + + +function FindSignTool { + $SignTool = "signtool.exe" + if (Get-Command $SignTool -ErrorAction SilentlyContinue) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\x86\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + $sdkVers = "10.0.22000.0", "10.0.20348.0", "10.0.19041.0", "10.0.17763.0" + Foreach ($ver in $sdkVers) + { + $SignTool = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\${ver}\x64\signtool.exe" + if (Test-Path -Path $SignTool -PathType Leaf) { + return $SignTool + } + } + "signtool.exe not found" + Exit 1 +} + +function SignEsptool { + param( + [Parameter()] + [String] + $Path + ) + + $SignTool = FindSignTool + "Using: $SignTool" + $CertificateFile = [system.io.path]::GetTempPath() + "certificate.pfx" + + if ($null -eq $env:CERTIFICATE) { + "CERTIFICATE variable not set, unable to sign the file" + Exit 1 + } + + if ("" -eq $env:CERTIFICATE) { + "CERTIFICATE variable is empty, unable to sign the file" + Exit 1 + } + + $SignParameters = @("sign", "/tr", 'http://timestamp.digicert.com', "/td", "SHA256", "/f", $CertificateFile, "/fd", "SHA256") + if ($env:CERTIFICATE_PASSWORD) { + "CERTIFICATE_PASSWORD detected, using the password" + $SignParameters += "/p" + $SignParameters += $env:CERTIFICATE_PASSWORD + } + $SignParameters += $Path + + [byte[]]$CertificateBytes = [convert]::FromBase64String($env:CERTIFICATE) + [IO.File]::WriteAllBytes($CertificateFile, $CertificateBytes) + + &$SignTool $SignParameters + + if (0 -eq $LASTEXITCODE) { + Remove-Item $CertificateFile + } else { + Remove-Item $CertificateFile + "Signing failed" + Exit 1 + } + +} + +SignEsptool ${Path} diff --git a/.github/pytools/espressif.ico b/.github/pytools/espressif.ico new file mode 100644 index 00000000000..c5550f148cf Binary files /dev/null and b/.github/pytools/espressif.ico differ diff --git a/.github/scripts/upload_py_tools.sh b/.github/scripts/upload_py_tools.sh new file mode 100755 index 00000000000..10c8416de44 --- /dev/null +++ b/.github/scripts/upload_py_tools.sh @@ -0,0 +1,11 @@ +#!/bin/bash +CHANGED_FILES=$1 +echo "Pushing '$CHANGED_FILES' as $GITHUB_ACTOR" +git config --global github.user "$GITHUB_ACTOR" +git config --global user.name "$GITHUB_ACTOR" +git config --global user.email "$GITHUB_ACTOR@users.noreply.github.com" +for tool in $CHANGED_FILES; do + git add tools/$tool.exe +done +git commit -m "Push binary to tools" +git push diff --git a/.github/workflows/build_py_tools.yml b/.github/workflows/build_py_tools.yml new file mode 100644 index 00000000000..907a3438bb8 --- /dev/null +++ b/.github/workflows/build_py_tools.yml @@ -0,0 +1,141 @@ +name: Build Python Tools + +on: + pull_request: + paths: + - 'tools/get.py' + - 'tools/espota.py' + - 'tools/gen_esp32part.py' + - 'tools/gen_insights_package.py' + +jobs: + find-changed-tools: + name: Check if tools have been changed + runs-on: ubuntu-20.04 + outputs: + any_changed: ${{ steps.verify-changed-files.outputs.any_changed }} + all_changed_files: ${{ steps.verify-changed-files.outputs.all_changed_files }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 2 + ref: ${{ github.event.pull_request.head.ref }} + - name: Verify Python Tools Changed + uses: tj-actions/changed-files@v36 + id: verify-changed-files + with: + fetch_depth: '2' + since_last_remote_commit: 'true' + files: | + tools/get.py + tools/espota.py + tools/gen_esp32part.py + tools/gen_insights_package.py + - name: List all changed files + shell: bash + run: | + for file in ${{ steps.verify-changed-files.outputs.all_changed_files }}; do + echo "$file was changed" + done + + build-pytools-binaries: + name: Build python tools binaries for ${{ matrix.os }} + runs-on: ${{ matrix.os }} + needs: find-changed-tools + if: needs.find-changed-tools.outputs.any_changed == 'true' + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest, ubuntu-20.04, ARM, ARM64] + include: + - os: windows-latest + TARGET: win64 + EXTEN: .exe + SEPARATOR: ';' + - os: macos-latest + TARGET: macos + SEPARATOR: ':' + - os: ubuntu-20.04 + TARGET: linux-amd64 + SEPARATOR: ':' + - os: ARM + CONTAINER: python:3.8-bullseye + TARGET: arm + SEPARATOR: ':' + - os: ARM64 + CONTAINER: python:3.8-bullseye + TARGET: arm64 + SEPARATOR: ':' + container: ${{ matrix.CONTAINER }} # use python container on ARM + env: + DISTPATH: pytools-${{ matrix.TARGET }} + PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi" + steps: + - name: List changed tools + shell: bash + run: | + CHANGED_FILES=() + for file in ${{ needs.find-changed-tools.outputs.all_changed_files }}; do + file="${file#*\/}" + file="${file%\.*}" + CHANGED_FILES+=("$file") + done + CHANGED_FILES="${CHANGED_FILES[@]}" + echo "CHANGED_TOOLS=$CHANGED_FILES" >> "$GITHUB_ENV" + for tool in ${{ env.CHANGED_TOOLS }}; do + echo "tool $tool was changed" + done + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.ref }} + - name: Set up Python 3.8 + # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108 + if: matrix.os != 'ARM' && matrix.os != 'ARM64' + uses: actions/setup-python@master + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pyinstaller requests + - name: Build with PyInstaller + shell: bash + run: | + for tool in ${{ env.CHANGED_TOOLS }}; do + pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=.github/pytools/espressif.ico tools/$tool.py + done + - name: Sign binaries + if: matrix.os == 'windows-latest' + env: + CERTIFICATE: ${{ secrets.CERTIFICATE }} + CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }} + shell: pwsh + run: | + $data = Write-Output ${{ env.CHANGED_TOOLS }} + foreach ( $node in $data ) + { + ./.github/pytools/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/$node.exe + } + - name: Test binaries + shell: bash + run: | + for tool in ${{ env.CHANGED_TOOLS }}; do + ./${{ env.DISTPATH }}/$tool${{ matrix.EXTEN }} -h + done + - name: Push binary to tools + if: matrix.os == 'windows-latest' + env: + GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} + shell: bash + run: | + for tool in ${{ env.CHANGED_TOOLS }}; do + cp -f ./${{ env.DISTPATH }}/$tool.exe tools/$tool.exe + done + bash .github/scripts/upload_py_tools.sh "${{ env.CHANGED_TOOLS }}" + - name: Archive artifact + uses: actions/upload-artifact@master + with: + name: ${{ env.DISTPATH }} + path: ${{ env.DISTPATH }} diff --git a/tools/espota.exe b/tools/espota.exe index 3de37913ce0..8ec76e15367 100644 Binary files a/tools/espota.exe and b/tools/espota.exe differ diff --git a/tools/get.exe b/tools/get.exe index a498e5214f3..542ac61baae 100644 Binary files a/tools/get.exe and b/tools/get.exe differ diff --git a/tools/get.py b/tools/get.py index 16e1e097b2a..10af3d14a9d 100755 --- a/tools/get.py +++ b/tools/get.py @@ -35,7 +35,13 @@ if 'Windows' in platform.system(): import requests -current_dir = os.path.dirname(os.path.realpath(unicode(__file__))) +# determine if application is a script file or frozen exe +if getattr(sys, 'frozen', False): + current_dir = os.path.dirname(os.path.realpath(unicode(sys.executable))) +elif __file__: + current_dir = os.path.dirname(os.path.realpath(unicode(__file__))) + +#current_dir = os.path.dirname(os.path.realpath(unicode(__file__))) dist_dir = current_dir + '/dist/' def sha256sum(filename, blocksize=65536): @@ -88,6 +94,8 @@ def unpack(filename, destination): # a little trick to rename tool directories so they don't contain version number rename_to = re.match(r'^([a-z][^\-]*\-*)+', dirname).group(0).strip('-') + if rename_to == dirname and dirname.startswith('esp32-arduino-libs-'): + rename_to = 'esp32-arduino-libs' if rename_to != dirname: print('Renaming {0} to {1} ...'.format(dirname, rename_to)) if os.path.isdir(rename_to): @@ -217,10 +225,16 @@ def identify_platform(): return arduino_platform_names[sys_name][bits] if __name__ == '__main__': + is_test = (len(sys.argv) > 1 and sys.argv[1] == '-h') + if is_test: + print('Test run!') identified_platform = identify_platform() print('Platform: {0}'.format(identified_platform)) tools_to_download = load_tools_list(current_dir + '/../package/package_esp32_index.template.json', identified_platform) mkdir_p(dist_dir) for tool in tools_to_download: - get_tool(tool) + if is_test: + print('Would install: {0}'.format(tool['archiveFileName'])) + else: + get_tool(tool) print('Platform Tools Installed')