Upstream changes

Jason2866 · Feb 1, 2024 · 5336fb6 · 5336fb6
1 parent 906f66e
commit 5336fb6
Show file tree

Hide file tree

Showing 24 changed files with 176 additions and 96 deletions.
diff --git a/.github/workflows/build_esptool.yml b/.github/workflows/build_esptool.yml
@@ -41,6 +41,12 @@ jobs:
           python -m pip install --upgrade pip
           pip install pyinstaller
           pip install --user -e .
+      - name: Build with PyInstaller
+        run: |
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.STUBS_DIR }}*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}" esptool.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.EFUSE_DIR }}*.yaml${{ matrix.SEPARATOR }}${{ env.EFUSE_DIR }}" espefuse.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico espsecure.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico esp_rfc2217_server.py
       - name: Build stub
         if: matrix.os == 'ubuntu-latest'
         run: |

diff --git a/.github/workflows/test_esptool.yml b/.github/workflows/test_esptool.yml
@@ -62,4 +62,15 @@ jobs:
         ./ci/setup_ci_build_env.sh
         make -C flasher_stub V=1
         cd flasher_stub && python ./compare_stubs.py
-  
+
+  lint_esptool:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@master
+
+        - name: Lint and format with ruff
+          run: |
+            pip install --extra-index-url https://dl.espressif.com/pypi -e .[dev]
+            python -m ruff check
+            python -m ruff format
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -41,6 +41,7 @@ cache:
     - host_test
   dependencies: []
   before_script:
+    - pip install --upgrade pip
     - pip install -e .[dev] --prefer-binary
 
 version_check:
@@ -128,12 +129,12 @@ check_python_style:
       codequality: code_quality_report.json
     when: on_failure
   script:
-    # This step installs any 'dev' dependencies (ie flake8, Black)
+    # This step installs any 'dev' dependencies (ie ruff)
     # The runner should cache the downloads, so still quite fast.
     - pip install -e .[dev] --prefer-binary
-    - python -m flake8 --exit-zero --format gl-codeclimate --output-file code_quality_report.json
-    - python -m flake8
-    - python -m black --check --diff .
+    - python -m ruff check --exit-zero --output-format=gitlab --output-file=code_quality_report.json
+    - python -m ruff check
+    - python -m ruff format
 
 .run_esptool: &run_esptool |
   esptool.py --help

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,13 +1,10 @@
 repos:
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
     hooks:
-      - id: flake8
-        additional_dependencies: [flake8-import-order]
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black
+      - id: ruff # Runs ruff linter (replaces flake8)
+        args: [--fix, --exit-non-zero-on-fix] # --fix for fixing errors
+      - id: ruff-format
   - repo: https://github.com/espressif/conventional-precommit-linter
     rev: v1.4.0
     hooks:

diff --git a/.ruff.toml b/.ruff.toml
@@ -0,0 +1,59 @@
+# https://docs.astral.sh/ruff/settings/
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".eggs",
+    ".git",
+    "__pycache__"
+]
+
+line-length = 88
+
+select = ['E', 'F', 'W']
+ignore = ["E203"]
+
+# Assume Python 3.7
+target-version = "py37"
+
+[per-file-ignores]
+
+
+# tests often manipulate sys.path before importing the main tools, so ignore import order violations
+"test/*.py" = ["E402"]
+
+# multiple spaces after ',' and long lines - used for visual layout of eFuse data
+"espefuse/efuse/*/mem_definition.py" =  ["E241", "E501"]
+"espefuse/efuse/*/operations.py" =   ["E241", "E501", "F401"]
+"espefuse/efuse/*/fields.py" =   ["E241", "E501"]
+
+# ignore long lines - used for RS encoding pairs
+"test/test_modules.py" = ["E501"]
+
+# don't check for unused imports in __init__.py files
+"__init__.py" = ["F401"]
+
+# allow definition from star imports in docs config
+"docs/conf_common.py" =  ["F405"]
+
+
+
+[lint]
+# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`)  codes by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E4", "E7", "E9", "F"]
+ignore = []
+
+# Allow fix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+
+# ruff-format hook configuration
+[format]
+quote-style = "double"
+indent-style = "space"
+docstring-code-format = true
+
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -85,24 +85,22 @@ To use the tool, first install ``pre-commit``. Then enable the ``pre-commit`` an
    $ python -m pip install pre-commit
    $ pre-commit install -t pre-commit -t commit-msg
 
-On the first commit ``pre-commit`` will install the hooks, subsequent checks will be significantly faster. If an error is found an appropriate error message will be displayed. If the error was with ``black`` then the tool will fix them for you automatically. Review the changes and re-stage for commit if you are happy with them.
+On the first commit ``pre-commit`` will install the hooks, subsequent checks will be significantly faster. If an error is found an appropriate error message will be displayed. Review the changes and re-stage for commit if you are happy with them.
 
 Conventional Commits
 """"""""""""""""""""
 
 ``esptool.py`` complies with the `Conventional Commits standard <https://www.conventionalcommits.org/en/v1.0.0/#specification>`_. Every commit message is checked with `Conventional Precommit Linter <https://github.com/espressif/conventional-precommit-linter>`_, ensuring it adheres to the standard.
 
-Flake8
-""""""
 
-``esptool.py`` complies with `flake8 <http://flake8.readthedocs.io/en/latest/>`_ style guide enforcement.
+Ruff
+""""
 
-Black
-"""""
+``esptool.py`` is `PEP8 <https://peps.python.org/pep-0008/>` compliant and enforces this style guide. For compliancy checking, we use `ruff <https://docs.astral.sh/ruff/>`.
+``Ruff`` also auto-format files in the same style as previously used ``black``.
 
-All files should be formatted using the `Black <https://black.readthedocs.io/en/stable/index.html>`_ auto-formatter.
 
-``Black``, ``flake8``, and ``Conventional Precommit Linter`` tools will be automatically run by ``pre-commit`` if that is configured. To check your code manually before submitting, run ``python -m flake8`` and ``black .`` (the tools are installed as part of the development requirements shown at the beginning of this document).
+``Ruff`` and ``Conventional Precommit Linter`` tools will be automatically run by ``pre-commit`` if that is configured. To check your code manually before submitting, run ``python -m ruff`` (this tool is installed as part of the development requirements shown at the beginning of this document).
 
 When you submit a Pull Request, the GitHub Actions automated build system will run automated checks using these tools.
 

diff --git a/docs/en/advanced-topics/boot-mode-selection.rst b/docs/en/advanced-topics/boot-mode-selection.rst
@@ -2,7 +2,7 @@
 
 {IDF_TARGET_STRAP_BOOT_2_GPIO:default="GPIO8", esp32="GPIO2", esp32s2="GPIO46", esp32s3="GPIO46"}
 
-{IDF_TARGET_BOOTLOADER_OFFSET:default="0",esp32="1000", esp32s2="1000"}
+{IDF_TARGET_BOOTLOADER_OFFSET:default="0", esp32="1000", esp32s2="1000", esp32p4="2000"}
 
 .. _boot-mode:
 

diff --git a/docs/en/esptool/advanced-commands.rst b/docs/en/esptool/advanced-commands.rst
@@ -1,4 +1,4 @@
-{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000"}
+{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000", esp32p4="0x2000"}
 
 .. _advanced-commands:
 

diff --git a/docs/en/esptool/flash-modes.rst b/docs/en/esptool/flash-modes.rst
@@ -1,4 +1,4 @@
-{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000"}
+{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000", esp32p4="0x2000"}
 
 {IDF_TARGET_FLASH_FREQ_F:default="80", esp32c2="60", esp32h2="48"}
 

diff --git a/docs/en/esptool/flashing-firmware.rst b/docs/en/esptool/flashing-firmware.rst
@@ -1,3 +1,5 @@
+{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000", esp32p4="0x2000"}
+
 .. _flashing:
 
 Flashing Firmware
@@ -42,10 +44,10 @@ It is also possible to assemble the command manually, please see the :ref:`espto
 ESP-IDF
 ^^^^^^^
 
-ESP-IDF outputs the full esptool command used for flashing after the build is finished::
+ESP-IDF outputs the full esptool command used for flashing after the build is finished, for example::
 
     Project build complete. To flash, run this command:
-    python esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip esp32  write_flash --flash_mode dio --flash_size detect --flash_freq 40m 0x1000 build/bootloader/bootloader.bin 0x8000 build/partition_table/partition-table.bin 0x10000 build/hello_world.bin
+    python esptool.py -p (PORT) -b 460800 --before default_reset --after hard_reset --chip {IDF_TARGET_PATH_NAME}  write_flash --flash_mode dio --flash_size detect --flash_freq 40m {IDF_TARGET_BOOTLOADER_OFFSET} build/bootloader/bootloader.bin 0x8000 build/partition_table/partition-table.bin 0x10000 build/hello_world.bin
     or run 'idf.py -p (PORT) flash'
 
 Arduino
@@ -56,11 +58,11 @@ The full esptool command is hidden from the user by default. To expose it, open
 PlatformIO
 ^^^^^^^^^^
 
-To do a verbose upload and see the exact esptool invocation, run ``pio run -v -t upload`` in the terminal. In the generated output, there is the full esptool command, e.g.:
+To do a verbose upload and see the exact esptool invocation, run ``pio run -v -t upload`` in the terminal. In the generated output, there is the full esptool command, you will see something like:
 
 ::
 
-    “.../.platformio/penv/bin/python2.7” “.../.platformio/packages/tool-esptoolpy/esptool.py” --chip esp32 --port “/dev/cu.usbserial001” --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dio --flash_freq 40m --flash_size detect 0x1000 .../.platformio/packages/framework-arduinoespressif32/tools/sdk/bin/bootloader_dio_40m.bin 0x8000 .../project_folder/.pio/build/esp32doit-devkit-v1/partitions.bin 0xe000 .../.platformio/packages/framework-arduinoespressif32/tools/partitions/boot_app0.bin 0x10000 .pio/build/esp32doit-devkit-v1/firmware.bin
+    “.../.platformio/penv/bin/python2.7” “.../.platformio/packages/tool-esptoolpy/esptool.py” --chip {IDF_TARGET_PATH_NAME} --port “/dev/cu.usbserial001” --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dio --flash_freq 40m --flash_size detect {IDF_TARGET_BOOTLOADER_OFFSET} .../.platformio/packages/framework-arduinoespressif32/tools/sdk/bin/bootloader_dio_40m.bin 0x8000 .../project_folder/.pio/build/esp32doit-devkit-v1/partitions.bin 0xe000 .../.platformio/packages/framework-arduinoespressif32/tools/partitions/boot_app0.bin 0x10000 .pio/build/esp32doit-devkit-v1/firmware.bin
 
 
 Flashing
@@ -70,7 +72,7 @@ If you split the output, you’ll find the ``write_flash`` command with a list o
 
 Change ``PORT`` to the name of :ref:`actually used serial port <serial-port>` and run the command. A successful flash looks like this::
 
-    $ python esptool.py -p /dev/tty.usbserial-0001 -b 460800 --before default_reset --after hard_reset --chip esp32  write_flash --flash_mode dio --flash_size detect --flash_freq 40m 0x1000 build/bootloader/bootloader.bin 0x8000 build/partition_table/partition-table.bin 0x10000 build/hello_world.bin
+    $ python esptool.py -p /dev/tty.usbserial-0001 -b 460800 --before default_reset --after hard_reset --chip {IDF_TARGET_PATH_NAME} write_flash --flash_mode dio --flash_size detect --flash_freq 40m {IDF_TARGET_BOOTLOADER_OFFSET} build/bootloader/bootloader.bin 0x8000 build/partition_table/partition-table.bin 0x10000 build/hello_world.bin
     esptool.py v3.2-dev
     Serial port /dev/tty.usbserial-0001
     Connecting.........

diff --git a/docs/en/troubleshooting.rst b/docs/en/troubleshooting.rst
@@ -1,4 +1,4 @@
-{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000"}
+{IDF_TARGET_BOOTLOADER_OFFSET:default="0x0", esp32="0x1000", esp32s2="0x1000", esp32p4="0x2000"}
 
 .. _troubleshooting:
 

diff --git a/espefuse/efuse/base_fields.py b/espefuse/efuse/base_fields.py
@@ -58,8 +58,9 @@ def check_arg_value(efuse, new_value):
             elif efuse.efuse_type.startswith("bytes"):
                 if new_value is None:
                     raise esptool.FatalError(
-                        "New value required for efuse '{}' "
-                        "(given None)".format(efuse.name)
+                        "New value required for efuse '{}' (given None)".format(
+                            efuse.name
+                        )
                     )
                 if len(new_value) * 8 != efuse.bitarray.len:
                     raise esptool.FatalError(

diff --git a/espefuse/efuse/esp32c2/fields.py b/espefuse/efuse/esp32c2/fields.py
@@ -368,15 +368,14 @@ def print_field(e, new_value):
 
 
 class EfuseKeyPurposeField(EfuseField):
+    # fmt: off
     KEY_PURPOSES = [
-        # fmt: off
         ("USER",                                        0, None),      # User purposes (software-only use)
         ("XTS_AES_128_KEY",                             1, None),      # (whole 256bits) flash/PSRAM encryption
         ("XTS_AES_128_KEY_DERIVED_FROM_128_EFUSE_BITS", 2, None),      # (lo 128bits) flash/PSRAM encryption
         ("SECURE_BOOT_DIGEST",                          3, "DIGEST"),
         # (hi 128bits) Secure Boot key digest
-        # fmt: on
-    ]
+    ]  # fmt: on
 
     KEY_PURPOSES_NAME = [name[0] for name in KEY_PURPOSES]
     DIGEST_KEY_PURPOSES = [name[0] for name in KEY_PURPOSES if name[2] == "DIGEST"]
diff --git a/espsecure/__init__.py b/espsecure/__init__.py
@@ -11,6 +11,7 @@
 import tempfile
 import zlib
 from collections import namedtuple
+from io import IOBase
 
 from cryptography import exceptions
 from cryptography.hazmat.backends import default_backend
@@ -1802,8 +1803,11 @@ def main(custom_commandline=None):
     finally:
         for arg_name in vars(args):
             obj = getattr(args, arg_name)
-            if isinstance(obj, OutFileType):
+            if isinstance(obj, (OutFileType, IOBase)):
                 obj.close()
+            elif isinstance(obj, list):
+                for f in [o for o in obj if isinstance(o, IOBase)]:
+                    f.close()
 
 
 def _main():

diff --git a/esptool/bin_image.py b/esptool/bin_image.py
@@ -1260,7 +1260,7 @@ def read_section_header(offs):
         nobits_secitons = [s for s in all_sections if s[1] == ELFFile.SEC_TYPE_NOBITS]
 
         # search for the string table section
-        if not (shstrndx * self.LEN_SEC_HEADER) in section_header_offsets:
+        if (shstrndx * self.LEN_SEC_HEADER) not in section_header_offsets:
             raise FatalError("ELF file has no STRTAB section at shstrndx %d" % shstrndx)
         _, sec_type, _, sec_size, sec_offs = read_section_header(
             shstrndx * self.LEN_SEC_HEADER

diff --git a/esptool/cmds.py b/esptool/cmds.py
@@ -815,9 +815,11 @@ def get_key_from_value(dict, val):
         print(
             "Checksum: {:#02x} ({})".format(
                 image.checksum,
-                "valid"
-                if image.checksum == calc_checksum
-                else "invalid - calculated {:02x}".format(calc_checksum),
+                (
+                    "valid"
+                    if image.checksum == calc_checksum
+                    else "invalid - calculated {:02x}".format(calc_checksum)
+                ),
             )
         )
         try:
@@ -892,8 +894,9 @@ def get_key_from_value(dict, val):
             ESP8266V2FirmwareImage.IMAGE_V2_MAGIC,
         ]:
             raise FatalError(
-                "This is not a valid image "
-                "(invalid magic number: {:#x})".format(magic)
+                "This is not a valid image " "(invalid magic number: {:#x})".format(
+                    magic
+                )
             )
 
         if args.chip == "auto":
@@ -940,9 +943,11 @@ def get_key_from_value(dict, val):
     print(
         "Checksum: {:02x} ({})".format(
             image.checksum,
-            "valid"
-            if image.checksum == calc_checksum
-            else "invalid - calculated {:02x}".format(calc_checksum),
+            (
+                "valid"
+                if image.checksum == calc_checksum
+                else "invalid - calculated {:02x}".format(calc_checksum)
+            ),
         )
     )
     try:

diff --git a/esptool/loader.py b/esptool/loader.py
@@ -300,7 +300,17 @@ def __init__(self, port=DEFAULT_PORT, baud=ESP_ROM_BAUD, trace_enabled=False):
 
         if isinstance(port, str):
             try:
-                self._port = serial.serial_for_url(port, exclusive=True)
+                self._port = serial.serial_for_url(
+                    port, exclusive=True, do_not_open=True
+                )
+                if sys.platform == "win32":
+                    # When opening a port on Windows,
+                    # the RTS/DTR (active low) lines
+                    # need to be set to False (pulled high)
+                    # to avoid unwanted chip reset
+                    self._port.rts = False
+                    self._port.dtr = False
+                self._port.open()
             except serial.serialutil.SerialException as e:
                 port_issues = [
                     [  # does not exist error
@@ -1641,12 +1651,14 @@ def __str__(self):
             while len(s) > 0:
                 line = s[:16]
                 ascii_line = "".join(
-                    c
-                    if (
-                        c == " "
-                        or (c in string.printable and c not in string.whitespace)
+                    (
+                        c
+                        if (
+                            c == " "
+                            or (c in string.printable and c not in string.whitespace)
+                        )
+                        else "."
                     )
-                    else "."
                     for c in line.decode("ascii", "replace")
                 )
                 s = s[16:]