Merge pull request #21 from Jason2866/merge

Merge

.github/ISSUE_TEMPLATE/bug-report-no-hw.yml

@@ -14,7 +14,7 @@ body:
     id: version
     attributes:
       label: Esptool Version
-      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider updating esptool.
+      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider [updating esptool](https://docs.espressif.com/projects/esptool/en/latest/installation.html#how-to-update).
       placeholder: ex. v4.0.1, commit v4.1-10-g2059335aa
     validations:
       required: true

.github/ISSUE_TEMPLATE/issue-with-hw.yml

@@ -21,7 +21,7 @@ body:
     id: version
     attributes:
       label: Esptool Version
-      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider updating esptool.
+      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider [updating esptool](https://docs.espressif.com/projects/esptool/en/latest/installation.html#how-to-update).
       placeholder: ex. v4.0.1, commit v4.1-10-g2059335aa
     validations:
       required: true

.gitlab-ci.yml

@@ -383,6 +383,14 @@ target_esp32h2_jtag_serial:
   script:
     - coverage run --parallel-mode -m pytest ${CI_PROJECT_DIR}/test/test_esptool.py --port /dev/serial_ports/ESP32H2_JTAG_SERIAL --preload-port /dev/serial_ports/ESP32H2_PRELOAD --chip esp32h2 --baud 115200
 
+# ESP32P4
+target_esp32p4:
+  extends: .target_esptool_test
+  tags:
+    - esptool_esp32p4_target
+  script:
+    - coverage run --parallel-mode -m pytest ${CI_PROJECT_DIR}/test/test_esptool.py --port /dev/serial_ports/ESP32P4 --chip esp32p4 --baud 115200
+
 .windows_test:
   stage: test
   variables:

docs/en/esptool/flashing-firmware.rst

@@ -1,3 +1,5 @@
+.. _flashing:
+
 Flashing Firmware
 =================
 

docs/en/installation.rst

@@ -1,8 +1,14 @@
-.. _installation:
-
 Installation and Dependencies
 =============================
 
+.. _installation:
+
+How to Install
+--------------
+
+Global Installation
+^^^^^^^^^^^^^^^^^^^
+
 You will need `Python 3.7 or newer <https://www.python.org/downloads/>`_ installed on your system to use the latest version of ``esptool.py``.
 If your use case requires Python 2.7, 3.4, 3.5, or 3.6, please use ``esptool.py`` v3.3.* instead.
 
@@ -21,3 +27,44 @@ After installing, you will have ``esptool.py`` installed into the default Python
 .. note::
 
    If you actually plan to do development work with esptool itself, see :ref:`development-setup` for more information.
+
+Virtual Environment Installation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To ensure that ``esptool.py`` is used in isolation, and any changes made during its usage won't affect other Python environments or SDK installations, it is advised to install it in a virtual environment and use it directly if possible (more information in the :ref:`flashing` article).
+
+Creating a virtual environment (venv) is a good practice. This is particularly helpful for users who may be concerned about interfering with existing installations (e.g. in an environment of a development-setup framework). Here's a quick guide:
+
+- Create a virtual environment and choose its name, e.g. 'esptoolenv': ``python -m venv esptoolenv``
+- Activate the virtual environment:
+
+   - On Windows: ``esptoolenv\Scripts\activate``
+   - On Linux or MacOS: ``source esptoolenv/bin/activate``
+
+- Install the latest ``esptool.py`` version within the active virtual environment: ``pip install esptool``
+- You can now use it within this virtual environment without affecting your system-wide installations: ``esptool.py <command>``
+- When you're done using ``esptool.py``, deactivate the virtual environment: ``deactivate``. The environment can be reused by activating it again.
+- If you no longer need the virtual environment, you can remove it by deleting the ``esptoolenv`` directory.
+
+How to Update
+-------------
+
+Standalone
+^^^^^^^^^^
+
+If you are using ``esptool.py`` as a standalone tool (as a global installation or in a virtual environment), updating to the latest version released on the `PyPI <https://pypi.org/project/esptool/>`_ index is simple:
+
+::
+
+   $ pip install --upgrade esptool
+
+As a Component
+^^^^^^^^^^^^^^
+
+If ``esptool.py`` is installed as a component of a development framework (e.g. `ESP-IDF <https://docs.espressif.com/projects/esp-idf/>`_, `Arduino <https://docs.espressif.com/projects/arduino-esp32/>`_, or `PlatformIO <https://docs.platformio.org/en/latest/platforms/espressif32.html>`_), it is advised to follow the update guide of used framework for instructions and not to update the tool directly.
+
+If updating directly is unavoidable, make sure you update to a compatible version by staying on the same MAJOR version number (explaned in the :ref:`versions` article). For instance, if your currently installed ``esptool.py`` is ``v3.3.1``, only update to ``v3.*.*``. You risk introducing incompatible changes by updating to ``v4.*.*`` or higher.
+
+::
+
+   $ pip install esptool==3.3.2

espefuse/__init__.py

@@ -214,6 +214,12 @@ def main(custom_commandline=None, esp=None):
         "Use with caution.",
         action="store_true",
     )
+    init_parser.add_argument(
+        "--postpone",
+        help="Postpone burning some efuses from BLOCK0 at the end, "
+        "(efuses which disable access to blocks or chip).",
+        action="store_true",
+    )
 
     common_args, remaining_args = init_parser.parse_known_args(custom_commandline)
     debug_mode = common_args.debug or ("dump" in remaining_args)
@@ -263,6 +269,8 @@ def main(custom_commandline=None, esp=None):
     if there_are_multiple_burn_commands_in_args:
         efuses.batch_mode_cnt += 1
 
+    efuses.postpone = common_args.postpone
+
     try:
         for rem_args in grouped_remaining_args:
             args, unused_args = parser.parse_known_args(rem_args, namespace=common_args)
@@ -289,6 +297,7 @@ def main(custom_commandline=None, esp=None):
             efuses.batch_mode_cnt -= 1
             if not efuses.burn_all(check_batch_mode=True):
                 raise esptool.FatalError("BURN was not done")
+            print("Successful")
     finally:
         if not external_esp and not common_args.virt and esp._port:
             esp._port.close()

espefuse/efuse/base_fields.py

@@ -445,6 +445,7 @@ class EspEfusesBase(object):
     coding_scheme = None
     force_write_always = None
     batch_mode_cnt = 0
+    postpone = False
 
     def __iter__(self):
         return self.efuses.__iter__()
@@ -489,6 +490,59 @@ def update_efuses(self):
         for efuse in self.efuses:
             efuse.update(self.blocks[efuse.block].bitarray)
 
+    def postpone_efuses_from_block0_to_burn(self, block):
+        postpone_efuses = {}
+
+        if block.id != 0:
+            return postpone_efuses
+
+        # We need to check this list of efuses. If we are going to burn an efuse
+        # from this list, then we need to split the burn operation into two
+        # steps. The first step involves burning efuses not in this list. In
+        # case of an error during this step, we can recover by burning the
+        # efuses from this list at the very end. This approach provides the
+        # ability to recover efuses if an error occurs during the initial burn
+        # operation.
+
+        # List the efuses here that must be burned at the very end, such as read
+        # and write protection fields, as well as efuses that disable
+        # communication with the espefuse tool.
+        efuses_list = ["WR_DIS", "RD_DIS"]
+        if self._esp.CHIP_NAME == "ESP32":
+            # Efuses below disables communication with the espefuse tool.
+            efuses_list.append("UART_DOWNLOAD_DIS")
+            # other efuses that are better to burn at the very end.
+            efuses_list.append("ABS_DONE_1")
+            efuses_list.append("FLASH_CRYPT_CNT")
+        else:
+            # Efuses below disables communication with the espefuse tool.
+            efuses_list.append("ENABLE_SECURITY_DOWNLOAD")
+            efuses_list.append("DIS_DOWNLOAD_MODE")
+            # other efuses that are better to burn at the very end.
+            efuses_list.append("SPI_BOOT_CRYPT_CNT")
+            efuses_list.append("SECURE_BOOT_EN")
+
+        def get_raw_value_from_write(self, efuse_name):
+            return self[efuse_name].get_bitstring(from_read=False)
+
+        for efuse_name in efuses_list:
+            postpone_efuses[efuse_name] = get_raw_value_from_write(self, efuse_name)
+
+        if any(value != 0 for value in postpone_efuses.values()):
+            if self.debug:
+                print("These BLOCK0 efuses will be burned later at the very end:")
+                print(postpone_efuses)
+            # exclude these efuses from the first burn (postpone them till the end).
+            for key_name in postpone_efuses.keys():
+                self[key_name].reset()
+        return postpone_efuses
+
+    def recover_postponed_efuses_from_block0_to_burn(self, postpone_efuses):
+        if any(value != 0 for value in postpone_efuses.values()):
+            print("Burn postponed efuses from BLOCK0.")
+            for key_name in postpone_efuses.keys():
+                self[key_name].save(postpone_efuses[key_name])
+
     def burn_all(self, check_batch_mode=False):
         if check_batch_mode:
             if self.batch_mode_cnt != 0:
@@ -508,18 +562,44 @@ def burn_all(self, check_batch_mode=False):
                 have_wr_data_for_burn = True
         if not have_wr_data_for_burn:
             print("Nothing to burn, see messages above.")
-            return
+            return True
         EspEfusesBase.confirm("", self.do_not_confirm)
 
-        # Burn from BLKn -> BLK0. Because BLK0 can set rd or/and wr protection bits.
-        for block in reversed(self.blocks):
+        def burn_block(block, postponed_efuses):
             old_fail = block.fail
             old_num_errors = block.num_errors
             block.burn()
             if (block.fail and old_fail != block.fail) or (
                 block.num_errors and block.num_errors > old_num_errors
             ):
+                if postponed_efuses:
+                    print("The postponed efuses were not burned due to an error.")
+                    print("\t1. Try to fix a coding error by this cmd:")
+                    print("\t   'espefuse.py check_error --recovery'")
+                    command_string = " ".join(
+                        f"{key} {value}"
+                        for key, value in postponed_efuses.items()
+                        if value.any(True)
+                    )
+                    print("\t2. Then run the cmd to burn all postponed efuses:")
+                    print(f"\t   'espefuse.py burn_efuse {command_string}'")
+
                 raise esptool.FatalError("Error(s) were detected in eFuses")
+
+        # Burn from BLKn -> BLK0. Because BLK0 can set rd or/and wr protection bits.
+        for block in reversed(self.blocks):
+            postponed_efuses = (
+                self.postpone_efuses_from_block0_to_burn(block)
+                if self.postpone
+                else None
+            )
+
+            burn_block(block, postponed_efuses)
+
+            if postponed_efuses:
+                self.recover_postponed_efuses_from_block0_to_burn(postponed_efuses)
+                burn_block(block, postponed_efuses)
+
         print("Reading updated efuses...")
         self.read_coding_scheme()
         self.read_blocks()
@@ -628,14 +708,21 @@ def convert_to_bitstring(self, new_value):
 
     def check_new_value(self, bitarray_new_value):
         bitarray_old_value = self.get_bitstring() | self.get_bitstring(from_read=False)
+
+        if not bitarray_new_value.any(True) and not bitarray_old_value.any(True):
+            return
+
         if bitarray_new_value.len != bitarray_old_value.len:
             raise esptool.FatalError(
                 "For {} efuse, the length of the new value is wrong, "
                 "expected {} bits, was {} bits.".format(
                     self.name, bitarray_old_value.len, bitarray_new_value.len
                 )
             )
-        if bitarray_new_value == bitarray_old_value:
+        if (
+            bitarray_new_value == bitarray_old_value
+            or bitarray_new_value & self.get_bitstring() == bitarray_new_value
+        ):
             error_msg = "\tThe same value for {} ".format(self.name)
             error_msg += "is already burned. Do not change the efuse."
             print(error_msg)
@@ -752,3 +839,14 @@ def get_info(self):
             if name is not None:
                 output += f"\n  Purpose: {self.parent[name].get()}\n "
         return output
+
+    def reset(self):
+        # resets a efuse that is prepared for burning
+        bitarray_field = self.convert_to_bitstring(0)
+        block = self.parent.blocks[self.block]
+        wr_bitarray_temp = block.wr_bitarray.copy()
+        position = wr_bitarray_temp.length - (
+            self.word * 32 + self.pos + bitarray_field.len
+        )
+        wr_bitarray_temp.overwrite(bitarray_field, pos=position)
+        block.wr_bitarray = wr_bitarray_temp

esptool/bin_image.py

@@ -55,14 +55,18 @@ def intel_hex_to_bin(file: BinaryIO, start_addr: Optional[int] = None) -> Binary
     INTEL_HEX_MAGIC = b":"
     magic = file.read(1)
     file.seek(0)
-    if magic == INTEL_HEX_MAGIC:
-        ih = IntelHex()
-        ih.loadhex(file.name)
-        file.close()
-        bin = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
-        ih.tobinfile(bin, start=start_addr)
-        return bin
-    else:
+    try:
+        if magic == INTEL_HEX_MAGIC:
+            ih = IntelHex()
+            ih.loadhex(file.name)
+            file.close()
+            bin = tempfile.NamedTemporaryFile(suffix=".bin", delete=False)
+            ih.tobinfile(bin, start=start_addr)
+            return bin
+        else:
+            return file
+    except HexRecordError:
+        # file started with HEX magic but the rest was not according to the standard
         return file
 
 

esptool/cmds.py

@@ -793,7 +793,7 @@ def get_key_from_value(dict, val):
         format_str = "{:7}  {:#07x}  {:#010x}  {:#010x}  {}"
         app_desc = None
         bootloader_desc = None
-        for idx, seg in enumerate(image.segments, start=1):
+        for idx, seg in enumerate(image.segments):
             segs = seg.get_memory_type(image)
             seg_name = ", ".join(segs)
             if "DROM" in segs:  # The DROM segment starts with the esp_app_desc_t struct

test/test_espefuse.py

@@ -1977,3 +1977,29 @@ def test_burn_ecdsa_key(self):
             "key due to a hardware bug (please see TRM for more details)",
             ret_code=2,
         )
+
+
+class TestPostponedEfuses(EfuseTestCase):
+    def test_postpone_efuses(self):
+        if arg_chip == "esp32":
+            cmd = f"--postpone \
+                    burn_efuse UART_DOWNLOAD_DIS 1 \
+                    burn_key BLOCK1 {IMAGES_DIR}/256bit \
+                    burn_efuse ABS_DONE_1 1 FLASH_CRYPT_CNT 1"
+            num = 1
+        else:
+            sb_digest_name = (
+                "SECURE_BOOT_DIGEST" if arg_chip == "esp32c2" else "SECURE_BOOT_DIGEST0"
+            )
+            cmd = f"--postpone \
+                burn_efuse ENABLE_SECURITY_DOWNLOAD 1 DIS_DOWNLOAD_MODE 1 \
+                SECURE_VERSION 1 \
+                burn_key BLOCK_KEY0 {IMAGES_DIR}/256bit {sb_digest_name} \
+                burn_efuse SPI_BOOT_CRYPT_CNT 1 SECURE_BOOT_EN 1"
+            num = 3 if arg_chip == "esp32c2" else 4
+        output = self.espefuse_py(cmd)
+        assert f"BURN BLOCK{num}  - OK" in output
+        assert "BURN BLOCK0  - OK" in output
+        assert "Burn postponed efuses from BLOCK0" in output
+        assert "BURN BLOCK0  - OK" in output
+        assert "Successful" in output

test/test_esptool.py

@@ -184,7 +184,7 @@ def run_esptool_process(cmd):
             preload
             and arg_preload_port
             and arg_chip
-            in ["esp32c3", "esp32s3", "esp32c6", "esp32h2"]  # With USB-JTAG/Serial
+            in ["esp32c3", "esp32s3", "esp32c6", "esp32h2", "esp32p4"]  # With U-JS
         ):
             port_index = base_cmd.index("--port") + 1
             base_cmd[port_index] = arg_preload_port  # Set the port to the preload one
@@ -1012,9 +1012,7 @@ class TestKeepImageSettings(EsptoolTestCase):
     def setup_class(self):
         super(TestKeepImageSettings, self).setup_class()
         self.BL_IMAGE = f"images/bootloader_{arg_chip}.bin"
-        self.flash_offset = (
-            0x1000 if arg_chip in ("esp32", "esp32s2") else 0
-        )  # bootloader offset
+        self.flash_offset = esptool.CHIP_DEFS[arg_chip].BOOTLOADER_FLASH_OFFSET
         with open(self.BL_IMAGE, "rb") as f:
             self.header = f.read(8)
 
@@ -1085,7 +1083,7 @@ def test_explicit_set_size_freq_mode(self):
 
 
 @pytest.mark.skipif(
-    arg_chip in ["esp32s2", "esp32s3"],
+    arg_chip in ["esp32s2", "esp32s3", "esp32p4"],
     reason="Not supported on targets with USB-CDC.",
 )
 class TestLoadRAM(EsptoolTestCase):
@@ -1169,7 +1167,7 @@ class TestBootloaderHeaderRewriteCases(EsptoolTestCase):
     )
     @pytest.mark.quick_test
     def test_flash_header_rewrite(self):
-        bl_offset = 0x1000 if arg_chip in ("esp32", "esp32s2") else 0
+        bl_offset = esptool.CHIP_DEFS[arg_chip].BOOTLOADER_FLASH_OFFSET
         bl_image = f"images/bootloader_{arg_chip}.bin"
 
         output = self.run_esptool(
@@ -1187,7 +1185,7 @@ def test_flash_header_rewrite(self):
     def test_flash_header_no_magic_no_rewrite(self):
         # first image doesn't start with magic byte, second image does
         # but neither are valid bootloader binary images for either chip
-        bl_offset = 0x1000 if arg_chip in ("esp32", "esp32s2") else 0
+        bl_offset = esptool.CHIP_DEFS[arg_chip].BOOTLOADER_FLASH_OFFSET
         for image in ["images/one_kb.bin", "images/one_kb_all_ef.bin"]:
             output = self.run_esptool(
                 f"write_flash -fm dout -ff 20m {bl_offset:#x} {image}"