Add rBoot partition support

.gitmodules

@@ -63,7 +63,7 @@
 	ignore = dirty
 [submodule "rboot"]
 	path = Sming/Components/rboot/rboot
-	url = https://github.com/raburton/rboot.git
+	url = https://github.com/mikee47/rboot
 	ignore = dirty
 [submodule "spiffs"]
 	path = Sming/Components/spiffs/spiffs

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -55,9 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)
 
 extern "C" uint32 ICACHE_FLASH_ATTR  WEAK_ATTR user_rf_cal_sector_set(void)
 {
-	// RF calibration stored in last sector of sysParam
-	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
-	return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
+	return rfCal.address();
 }
 
 #ifdef SDK_INTERNAL
@@ -71,16 +70,14 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 	Storage::initialize();
 
 	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
 	auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
 
-	// RF calibration stored in last sector of sysParam
-	auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;
-
 	static const partition_item_t partitions[] = {
 			{SYSTEM_PARTITION_BOOTLOADER,		0,						SPI_FLASH_SEC_SIZE},
 			{SYSTEM_PARTITION_PHY_DATA,			phy.address(),			phy.size()},
-			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParamSize},
-			{SYSTEM_PARTITION_RF_CAL,			sysParam.address() + sysParamSize,	SPI_FLASH_SEC_SIZE},
+			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParam.size()},
+			{SYSTEM_PARTITION_RF_CAL,			rfCal.address(),		rfCal.size()},
 	};
 
 	enum flash_size_map sizeMap = system_get_flash_size_map();

Sming/Arch/Esp8266/options.json

@@ -6,5 +6,24 @@
                 "filename": "$(FLASH_INIT_DATA_VCC)"
             }
         }
+    },
+    "legacy": {
+        "description": "Move partition table and config partitions into legacy locations",
+        "partition_table_offset": "self.devices[0].size - 0x6000",
+        "partitions": {
+            "rom0": {
+                "address": "0x2000",
+                "size": "0x0F8000"
+            },
+            "rf_cal": {
+                "address": "self.device.size - 0x5000"
+            },
+            "phy_init": {
+                "address": "self.device.size - 0x4000"
+            },
+            "sys_param": {
+                "address": "self.device.size - 0x3000"
+            }
+        }
     }
 }

Sming/Arch/Esp8266/spiffs-two-roms.hw

@@ -2,11 +2,14 @@
 	"name": "Two ROM slots with single SPIFFS",
 	"base_config": "spiffs",
 	"partitions": {
+		"rom0": {
+			"subtype": "ota_0"
+		},
 		"rom1": {
 			"address": "0x108000",
 			"size": "992K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Arch/Esp8266/standard.hw

@@ -21,10 +21,16 @@
 		},
 		"sys_param": {
 			"address": "0x004000",
-			"size": "16K",
+			"size": "12K",
 			"type": "data",
 			"subtype": "sysparam"
 		},
+		"rf_cal": {
+			"address": "0x007000",
+			"size": "4K",
+			"type": "data",
+			"subtype": "rfcal"
+		},
 		"rom0": {
 			"address": "0x008000",
 			"size": "992K",

Sming/Arch/Esp8266/two-rom-mode.hw

@@ -3,13 +3,14 @@
 	"base_config": "standard",
 	"partitions": {
 		"rom0": {
+			"subtype": "ota_0",
 			"size": "480K"
 		},
 		"rom1": {
 			"address": "0x80000",
-			"size": "512K",
+			"size": "488K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Components/Storage/README.rst

@@ -84,6 +84,34 @@ If using this approach, remember to updated your project's ``component.mk`` with
 and verify the layout is correct using ``make map``.
 
 
+Legacy OTA updates
+------------------
+
+Because the standard flash layout has changed, performing OTA updates from Sming 4.2 can be tricky.
+There are a several ways to handle this, but the simplest is to retain the existing partition layout.
+
+You can use the ``legacy`` option to move the partition table and configuration partitions
+back to their original locations::
+
+   make map HWCONFIG_OPTS=legacy
+
+When using this approach, remember to add ``HWCONFIG_OPTS := legacy`` to your project's component.mk file.
+
+If you need to make any further adjustments, see below for how to create a custom profile.
+You can use the ``legacy`` option in your profile like this::
+
+.. code-block:: json
+
+   "options": ["legacy"]
+
+
+Check that the displayed partition map corresponds to your project.
+For example, the partition table requires a free sector so must not overlap other partitions.
+
+Your OTA update process must include a step to write the partition table to the correct location.
+
+It is not necessary to update the bootloader. See :component:`rboot` for further information.
+
 
 Custom configurations
 ---------------------
@@ -356,7 +384,7 @@ you can take advantage of the partition API to manage them as follows:
 -  Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice`
    in your ``init()`` function (or elsewhere if more appropriate).
 
- 
+
 API
 ---
 

Sming/Components/Storage/Tools/hwconfig/config.py

@@ -46,6 +46,7 @@ def from_name(cls, name):
         config.load(name)
         if options != '':
             config.parse_options(options.split(','))
+        config.resolve_expressions()
         config.partitions.sort()
         return config
 
@@ -73,6 +74,9 @@ def parse_options(self, options):
             temp.pop('description', None)
             self.parse_dict(temp)
 
+    def resolve_expressions(self):
+        self.partitions.offset = eval(str(self.partitions.offset))
+
     def parse_dict(self, data):
         base_config = data.pop('base_config', None)
         if base_config is not None:
@@ -89,7 +93,7 @@ def parse_dict(self, data):
             elif k == 'arch':
                 self.arch = v
             elif k == 'partition_table_offset':
-                self.partitions.offset = parse_int(v)
+                self.partitions.offset = v
             elif k == 'devices':
                 self.devices.parse_dict(v)
             elif k == 'comment':
@@ -132,7 +136,7 @@ def buildVars(self):
         return res
 
     def verify(self, secure):
-        self.partitions.verify(self.arch, secure)
+        self.partitions.verify(self.arch, self.devices[0], secure)
 
     def map(self):
         return partition.Map(self.partitions, self.devices)

Sming/Components/Storage/Tools/hwconfig/hwconfig.py

@@ -44,7 +44,7 @@ def handle_flashcheck(args, config, part):
     for e in list:
         addr, filename = e.split('=')
         addr = int(addr, 0)
-        part = config.partitions.find_by_address(addr)
+        part = config.partitions.find_by_address(config.devices[0], addr)
         if part is None:
             raise InputError("No partition contains address 0x%08x" % addr)
         if part.address != addr:
@@ -113,5 +113,5 @@ def main():
     try:
         main()
     except InputError as e:
-        print(e, file=sys.stderr)
+        print("** ERROR! %s" % e, file=sys.stderr)
         sys.exit(2)

Sming/Components/Storage/Tools/hwconfig/partition.py

@@ -16,11 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import struct, hashlib, storage, binascii
+import struct, hashlib, storage, binascii, copy
 from common import *
 
 MAX_PARTITION_LENGTH = 0xC00  # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14  # The first 2 bytes are like magic numbers for MD5 sum
+FLASH_SECTOR_SIZE = 0x1000
 PARTITION_TABLE_SIZE = 0x1000  # Size of partition table
 PARTITION_ENTRY_SIZE = 32
 
@@ -84,6 +85,7 @@
         "nvs_keys": 0x04,
         "efuse": 0x05,
         "sysparam": 0x40,
+        "rfcal": 0x41,
         "esphttpd": 0x80,
         "fat": 0x81,
         "spiffs": 0x82,
@@ -198,19 +200,28 @@ def find_by_name(self, name):
                 return p
         return None
 
-    def find_by_address(self, addr):
+    def find_by_address(self, device, addr):
         for p in self:
-            if p.contains(addr):
+            if p.device == device and p.contains(addr):
                 return p
         return None
 
-    def verify(self, arch, secure):
+    def verify(self, arch, spiFlash, secure):
         """Verify partition layout
         """
         # verify each partition individually
         for p in self:
             p.verify(arch, secure)
 
+        if self.offset % FLASH_SECTOR_SIZE != 0:
+            raise InputError("Partition table offset not aligned to flash sector")
+
+        p = self.find_by_address(spiFlash, self.offset)
+        if p is None:
+            p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1)
+        if not p is None:
+            raise InputError("Partition table conflict with '%s'" % p.name)
+
         # check on duplicate name
         names = [p.name for p in self]
         duplicates = set(n for n in names if names.count(n) > 1)
@@ -224,7 +235,10 @@ def verify(self, arch, secure):
             raise InputError("Partition names must be unique")
 
         # check for overlaps
-        minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        if arch == 'Esp32':
+            minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        else:
+            minPartitionAddress = 0x00002000
         dev = ''
         last = None
         for p in self:
@@ -345,7 +359,7 @@ def parse_dict(self, data, devices):
                 if k == 'device':
                     self.device = devices.find_by_name(v)
                 elif k == 'address':
-                    self.address = parse_int(v)
+                    self.address = eval(str(v))
                 elif k == 'size':
                     self.size = parse_int(v)
                 elif k == 'filename':
@@ -523,13 +537,17 @@ def add_unused(address, last_end):
             if address > last_end + 1:
                 add('(unused)', last_end + 1, address - last_end - 1)
 
+        partitions = copy.copy(table)
+
         if table.offset == 0:
             last = None
         else:
-            add("Boot Sector", 0, table.offset)
-            last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE)
+            last = add('Boot Sector', 0, min(table.offset, partitions[0].address))
+            p = Entry(device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, 0xff, 0xff)
+            partitions.append(p)
+            partitions.sort()
 
-        for p in table:
+        for p in partitions:
             if last is not None:
                 if p.device != last.device:
                     add_unused(last.device.size, last.end())

Sming/Components/Storage/component.mk

@@ -47,6 +47,7 @@ else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW))
 $(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH)))
 endif
 COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET)
+COMPONENT_CPPFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET)
 
 # Function to evaluate expression against config
 HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part "$(PART)") expr $(HWCONFIG) -

Sming/Components/Storage/src/include/Storage/Partition.h

@@ -59,6 +59,7 @@
 	XX(nvsKeys, 0x04, "NVS key information")                                                                           \
 	XX(eFuseEm, 0x05, "eFuse emulation")                                                                               \
 	XX(sysParam, 0x40, "System Parameters")                                                                            \
+	XX(rfCal, 0x41, "RF Calibration")                                                                                  \
 	XX(espHttpd, 0x80, "ESPHTTPD")                                                                                     \
 	XX(fat, 0x81, "FAT")                                                                                               \
 	XX(spiffs, 0x82, "SPIFFS")                                                                                         \

Sming/Components/Storage/src/include/Storage/PartitionTable.h

@@ -45,7 +45,7 @@ class PartitionTable
 
 	/**
 	 * @brief Find partition by name
-	 * @param Name to search for, case-sensitive
+	 * @param Name Name to search for, case-sensitive
 	 * @retval Partition
 	 *
 	 * Names are unique so at most only one match
@@ -55,6 +55,26 @@ class PartitionTable
 		return *std::find(begin(), end(), name);
 	}
 
+	/**
+	 * @brief Find partition containing the given address
+	 * @param address Address to search for
+	 * @retval Partition
+	 */
+	Partition find(uint32_t address) const
+	{
+		return *std::find_if(begin(), end(), [address](Partition part) { return part.contains(address); });
+	}
+
+	/**
+	 * @brief Find the n'th OTA partition
+	 */
+	Partition findOta(uint8_t index)
+	{
+		using App = Partition::SubType::App;
+		auto subtype = App(uint8_t(App::ota0) + index);
+		return (subtype >= App::ota_min && subtype <= App::ota_max) ? *find(subtype) : Partition{};
+	}
+
 	Iterator begin() const
 	{
 		return Iterator(mDevice, 0);