Add rBoot partition support

.gitmodules

@@ -63,7 +63,7 @@
 	ignore = dirty
 [submodule "rboot"]
 	path = Sming/Components/rboot/rboot
-	url = https://github.com/raburton/rboot.git
+	url = https://github.com/mikee47/rboot
 	ignore = dirty
 [submodule "spiffs"]
 	path = Sming/Components/spiffs/spiffs

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -55,9 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)
 
 extern "C" uint32 ICACHE_FLASH_ATTR  WEAK_ATTR user_rf_cal_sector_set(void)
 {
-	// RF calibration stored in last sector of sysParam
-	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
-	return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
+	return rfCal.address();
 }
 
 #ifdef SDK_INTERNAL
@@ -71,16 +70,14 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 	Storage::initialize();
 
 	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
 	auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
 
-	// RF calibration stored in last sector of sysParam
-	auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;
-
 	static const partition_item_t partitions[] = {
 			{SYSTEM_PARTITION_BOOTLOADER,		0,						SPI_FLASH_SEC_SIZE},
 			{SYSTEM_PARTITION_PHY_DATA,			phy.address(),			phy.size()},
-			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParamSize},
-			{SYSTEM_PARTITION_RF_CAL,			sysParam.address() + sysParamSize,	SPI_FLASH_SEC_SIZE},
+			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParam.size()},
+			{SYSTEM_PARTITION_RF_CAL,			rfCal.address(),		rfCal.size()},
 	};
 
 	enum flash_size_map sizeMap = system_get_flash_size_map();

Sming/Arch/Esp8266/options.json

@@ -6,5 +6,23 @@
                 "filename": "$(FLASH_INIT_DATA_VCC)"
             }
         }
+    },
+    "alternate": {
+        "description": "ESP8266 layout with critical partitions at start of flash",
+        "partition_table_offset": "0x2000",
+        "partitions": {
+            "rf_cal": {
+                "address": "0x3000"
+            },
+            "phy_init": {
+                "address": "0x4000"
+            },
+            "sys_param": {
+                "address": "0x5000"
+            },
+            "rom0": {
+                "address": "0x8000"
+            }
+        }
     }
 }

Sming/Arch/Esp8266/spiffs-two-roms.hw

@@ -2,11 +2,14 @@
 	"name": "Two ROM slots with single SPIFFS",
 	"base_config": "spiffs",
 	"partitions": {
+		"rom0": {
+			"subtype": "ota_0"
+		},
 		"rom1": {
 			"address": "0x108000",
 			"size": "992K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Arch/Esp8266/standard.hw

@@ -2,7 +2,7 @@
 	"name": "Standard config with single ROM",
 	"comment": "Should work with any Esp8266 variant",
 	"arch": "Esp8266",
-	"partition_table_offset": "0x2000",
+	"partition_table_offset": "self.devices[0].size - 0x6000",
 	"devices": {
 		"spiFlash": {
 			"type": "flash",
@@ -12,25 +12,31 @@
 		}
 	},
 	"partitions": {
+		"rom0": {
+			"address": "0x002000",
+			"size": "992K",
+			"type": "app",
+			"subtype": "factory",
+			"filename": "$(RBOOT_ROM_0_BIN)"
+		},
+		"rf_cal": {
+			"address": "self.device.size - 0x5000",
+			"size": "4K",
+			"type": "data",
+			"subtype": "rfcal"
+		},
 		"phy_init": {
-			"address": "0x003000",
+			"address": "self.device.size - 0x4000",
 			"size": "4K",
 			"type": "data",
 			"subtype": "phy",
 			"filename": "$(FLASH_INIT_DATA)"
 		},
 		"sys_param": {
-			"address": "0x004000",
-			"size": "16K",
+			"address": "self.device.size - 0x3000",
+			"size": "12K",
 			"type": "data",
 			"subtype": "sysparam"
-		},
-		"rom0": {
-			"address": "0x008000",
-			"size": "992K",
-			"type": "app",
-			"subtype": "factory",
-			"filename": "$(RBOOT_ROM_0_BIN)"
 		}
 	}
-}
+}

Sming/Arch/Esp8266/two-rom-mode.hw

@@ -3,13 +3,14 @@
 	"base_config": "standard",
 	"partitions": {
 		"rom0": {
+			"subtype": "ota_0",
 			"size": "480K"
 		},
 		"rom1": {
 			"address": "0x80000",
-			"size": "512K",
+			"size": "488K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Components/Storage/README.rst

@@ -84,6 +84,16 @@ If using this approach, remember to updated your project's ``component.mk`` with
 and verify the layout is correct using ``make map``.
 
 
+OTA updates
+-----------
+
+When planning OTA updates please check that the displayed partition map corresponds to your project.
+For example, the partition table requires a free sector so must not overlap other partitions.
+
+Your OTA update process must include a step to write the partition table to the correct location.
+
+It is not necessary to update the bootloader. See :component:`rboot` for further information.
+
 
 Custom configurations
 ---------------------
@@ -173,6 +183,19 @@ To customise the hardware configuration for a project, for example 'my_project':
    This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry).
 
 
+.. note::
+
+   The build system isn't smart enough to track dependencies for partition build targets.
+
+   To rebuild these manually type::
+
+      make partbuild
+
+   These will be removed when ``make clean`` is run, but you can also clean them separately thus::
+
+      make part-clean
+
+
 Partition maps
 --------------
 
@@ -356,7 +379,7 @@ you can take advantage of the partition API to manage them as follows:
 -  Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice`
    in your ``init()`` function (or elsewhere if more appropriate).
 
- 
+
 API
 ---
 

Sming/Components/Storage/Tools/hwconfig/config.py

@@ -46,6 +46,7 @@ def from_name(cls, name):
         config.load(name)
         if options != '':
             config.parse_options(options.split(','))
+        config.resolve_expressions()
         config.partitions.sort()
         return config
 
@@ -73,6 +74,9 @@ def parse_options(self, options):
             temp.pop('description', None)
             self.parse_dict(temp)
 
+    def resolve_expressions(self):
+        self.partitions.offset = eval(str(self.partitions.offset))
+
     def parse_dict(self, data):
         base_config = data.pop('base_config', None)
         if base_config is not None:
@@ -89,7 +93,7 @@ def parse_dict(self, data):
             elif k == 'arch':
                 self.arch = v
             elif k == 'partition_table_offset':
-                self.partitions.offset = parse_int(v)
+                self.partitions.offset = v
             elif k == 'devices':
                 self.devices.parse_dict(v)
             elif k == 'comment':
@@ -132,7 +136,7 @@ def buildVars(self):
         return res
 
     def verify(self, secure):
-        self.partitions.verify(self.arch, secure)
+        self.partitions.verify(self.arch, self.devices[0], secure)
 
     def map(self):
         return partition.Map(self.partitions, self.devices)

Sming/Components/Storage/Tools/hwconfig/hwconfig.py

@@ -44,7 +44,7 @@ def handle_flashcheck(args, config, part):
     for e in list:
         addr, filename = e.split('=')
         addr = int(addr, 0)
-        part = config.partitions.find_by_address(addr)
+        part = config.partitions.find_by_address(config.devices[0], addr)
         if part is None:
             raise InputError("No partition contains address 0x%08x" % addr)
         if part.address != addr:
@@ -113,5 +113,5 @@ def main():
     try:
         main()
     except InputError as e:
-        print(e, file=sys.stderr)
+        print("** ERROR! %s" % e, file=sys.stderr)
         sys.exit(2)

Sming/Components/Storage/Tools/hwconfig/partition.py

@@ -16,11 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import struct, hashlib, storage, binascii
+import struct, hashlib, storage, binascii, copy
 from common import *
 
 MAX_PARTITION_LENGTH = 0xC00  # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14  # The first 2 bytes are like magic numbers for MD5 sum
+FLASH_SECTOR_SIZE = 0x1000
 PARTITION_TABLE_SIZE = 0x1000  # Size of partition table
 PARTITION_ENTRY_SIZE = 32
 
@@ -84,6 +85,7 @@
         "nvs_keys": 0x04,
         "efuse": 0x05,
         "sysparam": 0x40,
+        "rfcal": 0x41,
         "esphttpd": 0x80,
         "fat": 0x81,
         "spiffs": 0x82,
@@ -154,6 +156,10 @@ def offset_str(self):
     def buildVars(self):
         dict = {}
         dict['PARTITION_NAMES'] = " ".join(p.name for p in self)
+        buildparts = [p for p in self if p.build is not None]
+        dict['PARTITIONS_WITH_TARGETS'] = " ".join(p.name for p in buildparts)
+        dict['PARTITION_BUILD_TARGETS'] = " ".join(p.filename for p in buildparts)
+
         for p in self:
             dict.update(p.buildVars())
         return dict
@@ -198,19 +204,28 @@ def find_by_name(self, name):
                 return p
         return None
 
-    def find_by_address(self, addr):
+    def find_by_address(self, device, addr):
         for p in self:
-            if p.contains(addr):
+            if p.device == device and p.contains(addr):
                 return p
         return None
 
-    def verify(self, arch, secure):
+    def verify(self, arch, spiFlash, secure):
         """Verify partition layout
         """
         # verify each partition individually
         for p in self:
             p.verify(arch, secure)
 
+        if self.offset % FLASH_SECTOR_SIZE != 0:
+            raise InputError("Partition table offset not aligned to flash sector")
+
+        p = self.find_by_address(spiFlash, self.offset)
+        if p is None:
+            p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1)
+        if not p is None:
+            raise InputError("Partition table conflict with '%s'" % p.name)
+
         # check on duplicate name
         names = [p.name for p in self]
         duplicates = set(n for n in names if names.count(n) > 1)
@@ -224,7 +239,10 @@ def verify(self, arch, secure):
             raise InputError("Partition names must be unique")
 
         # check for overlaps
-        minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        if arch == 'Esp32':
+            minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        else:
+            minPartitionAddress = 0x00002000
         dev = ''
         last = None
         for p in self:
@@ -345,7 +363,7 @@ def parse_dict(self, data, devices):
                 if k == 'device':
                     self.device = devices.find_by_name(v)
                 elif k == 'address':
-                    self.address = parse_int(v)
+                    self.address = eval(str(v))
                 elif k == 'size':
                     self.size = parse_int(v)
                 elif k == 'filename':
@@ -523,13 +541,17 @@ def add_unused(address, last_end):
             if address > last_end + 1:
                 add('(unused)', last_end + 1, address - last_end - 1)
 
+        partitions = copy.copy(table)
+
         if table.offset == 0:
             last = None
         else:
-            add("Boot Sector", 0, table.offset)
-            last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE)
+            last = add('Boot Sector', 0, min(table.offset, partitions[0].address))
+            p = Entry(device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, 0xff, 0xff)
+            partitions.append(p)
+            partitions.sort()
 
-        for p in table:
+        for p in partitions:
             if last is not None:
                 if p.device != last.device:
                     add_unused(last.device.size, last.end())