Partition tables (#2171)

!!! WARNING: BREAKING CHANGE !!!

SPI_SIZE and DISABLE_SPIFFS directives are removed! 
If you have an application that is using spiffs make sure to add the following line in your component.mk and remove the `DISABLE_SPIFFS` directive, if present.

```
HWCONFIG := spiffs
```

If your application is not using spiffs then just remove the `DISABLE_SPIFFS` directive.

For more information take a look at the [migration document](https://sming.readthedocs.io/en/feature-partition-tables/upgrading/4.2-4.3.html).

More information about the changes can be seen below:
==========================================

Implement class-based API for Sming with support for all architectures.
See #1676 for discussion.

Summary of changes:

* Create new `Storage` component and port low-level Esp32 partition support
* Use `gen_esp32part.py` tool as basis for new `hwconfig.py` tool. Integrate into build system. Add new targets.
* Implement a C++ API. `Partition::SubType::App` and `Partition::SubType::Data` are strong enums used to identify all the standard types, but the base API still uses type/subtype so custom types can be accommodated.
* Revise esp8266 sector layout - see updated documentation. This places all the critical information at the start of flash, whereas previously it was at the end. This avoids issues where flash size is set incorrectly.
* Custom `sysParam` data partition sub-type used for esp8266 system parameters and RF calibration data.
* Add support for 'external' devices (e.g. SPIRAM, supplementary flash, EEPROM, etc.) using a custom `Storage` partition type. Applications register a `Storage::Device` object implementation with the Partition API to register these external devices with a C++ interface for them.
* Disentangle SPIFFS from rBoot - OTA upgrading uses partition API.
* Drive SPIFFS image creation from partition table entries - more flexible, multiple images.
* Generate build variables directly from hardware config file (see `out/.../hwconfig.mk`). Preserve existing build variables where possible, document changes.
* Document hardware configuration with schema and incorporate validation. This can assist with fixing malformed configurations and supplements checks included in the partition tool.
* Add `Basic_Storage` sample to more thoroughly demonstrate usage
* Update documentation
* Test (HostTests) on all arches
* Remove `DISABLE_SPIFFS`, `SPI_SIZE` and `SPIFF_SIZE` build variables from all samples, including those in submodules.
* Include samples and tests within submodules to CI
* Add check to ensure image size will fit within partition

.ci/install.cmd

@@ -1,6 +1,6 @@
 REM Windows install script
 
-python -m pip install --upgrade pip
+python -m pip install --upgrade pip -r %SMING_HOME%\..\Tools\requirements.txt
 
 rmdir /s /q c:\MinGW
 curl -Lo MinGW.7z %SMINGTOOLS%/MinGW-2020-10-19.7z

.ci/install.sh

@@ -6,7 +6,7 @@ set -ex # exit with nonzero exit code if anything fails
 sudo apt-get update
 
 sudo update-alternatives --set gcc /usr/bin/gcc-9
-python -m pip install --upgrade pip
+python -m pip install --upgrade pip -r $SMING_HOME/../Tools/requirements.txt
 
 sudo apt-get install -y gcc-9-multilib g++-9-multilib python3-setuptools
 

.github/workflows/codeql-analysis.yml

@@ -36,11 +36,6 @@ jobs:
         # a pull request then we can checkout the head.
         fetch-depth: 2
 
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v1
@@ -68,13 +63,14 @@ jobs:
       run: |
        sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
        sudo apt-get update -y
-       sudo apt-get install gcc-9-multilib g++-9-multilib
+       sudo apt-get install gcc-9-multilib g++-9-multilib python3-wheel python3-setuptools
        sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9
+       python3 -m pip install --upgrade pip wheel -r Tools/requirements.txt
        env
        cd Sming
        export SMING_HOME=$(pwd)
        cd ../samples/Basic_Blink
-       make SMING_ARCH=Host
+       make -j3 SMING_ARCH=Host
        
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v1

Sming/Arch/Esp32/Components/esp32/src/startup.cpp

@@ -20,6 +20,7 @@
 #include <Platform/System.h>
 #include <driver/hw_timer.h>
 #include <driver/uart.h>
+#include <Storage.h>
 
 #ifndef ESP32_STACK_SIZE
 #define ESP32_STACK_SIZE 16384U
@@ -63,6 +64,7 @@ void main(void*)
 	esp_init_flash();
 	esp_init_wifi();
 	ets_init_tasks();
+	Storage::initialize();
 	System.initialize();
 	init();
 

Sming/Arch/Esp32/Components/sming-arch/component.mk

@@ -18,7 +18,6 @@ COMPONENT_DEPENDS := \
 	driver \
 	heap \
 	fatfs \
-	esp_spiffs \
 	esp32 \
 	gdbstub \
 	esptool

Sming/Arch/Esp32/app.mk

@@ -42,52 +42,3 @@ $(TARGET_OUT): $(COMPONENTS_AR)
 
 $(TARGET_BIN): $(TARGET_OUT)
 	$(Q) $(ESPTOOL_CMDLINE) elf2image --min-rev 0 --elf-sha256-offset 0xb0 $(flashimageoptions) -o $@ $<
-
-##@Flashing
-
-# Partitions
-PARTITIONS_CSV ?= $(BUILD_BASE)/partitions.csv
-PARTITIONS_BIN = $(FW_BASE)/partitions.bin
-
-CUSTOM_TARGETS += $(PARTITIONS_CSV)
-
-$(BUILD_BASE)/partitions.csv: | $(BUILD_BASE)
-	$(Q) cp $(SDK_PARTITION_PATH)/base.csv $@
-	@echo "storage, data, spiffs, $(SPIFF_START_ADDR), $(SPIFF_SIZE)," >> $@
-
-$(PARTITIONS_BIN): $(PARTITIONS_CSV)
-	$(Q) $(ESP32_PYTHON) $(SDK_COMPONENTS_PATH)/partition_table/gen_esp32part.py $< $@
-
-.PHONY: partitions
-partitions: $(PARTITIONS_BIN) ##Generate partitions table
-
-
-FLASH_PARTITION_CHUNKS	:= 0x8000=$(PARTITIONS_BIN)
-
-# Application
-
-FLASH_APP_CHUNKS := 0x10000=$(TARGET_BIN)
-
-.PHONY: flashboot
-flashboot: $(FLASH_BOOT_LOADER) ##Write just the Bootloader
-	$(call WriteFlash,$(FLASH_BOOT_CHUNKS))
-
-.PHONY: flashconfig
-
-flashconfig: partitions kill_term ##Write partition config
-	$(call WriteFlash,$(FLASH_PARTITION_CHUNKS))
-
-flashpartition: flashconfig
-
-.PHONY: flashapp
-flashapp: all kill_term ##Write just the application image
-	$(call WriteFlash,$(FLASH_APP_CHUNKS))
-
-.PHONY: flash
-flash: all partitions kill_term ##Write the boot loader, application image, partition table and (if enabled) SPIFFS image
-	$(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_APP_CHUNKS) $(FLASH_PARTITION_CHUNKS) $(FLASH_SPIFFS_CHUNKS))
-ifeq ($(ENABLE_GDB), 1)
-	$(GDB_CMDLINE)
-else
-	$(TERMINAL)
-endif

Sming/Arch/Esp32/standard.hw

@@ -0,0 +1,35 @@
+{
+	"name": "Standard config with single ROM",
+	"comment": "Should work with any Esp32 variant",
+	"arch": "Esp32",
+	"partition_table_offset": "0x8000",
+	"devices": {
+		"spiFlash": {
+			"type": "flash",
+			"size": "4M",
+			"mode": "dio",
+			"speed": 40
+		}
+	},
+	"partitions": {
+		"phy_init": {
+			"address": "0x00f000",
+			"size": "0x1000",
+			"type": "data",
+			"subtype": "phy"
+		},
+		"nvs": {
+			"address": "0x009000",
+			"size": "0x6000",
+			"type": "data",
+			"subtype": "nvs"
+		},
+		"factory": {
+			"address": "0x010000",
+			"size": "0x1f0000",
+			"type": "app",
+			"subtype": "factory",
+			"filename": "$(TARGET_BIN)"
+		}
+	}
+}

Sming/Arch/Esp8266/Components/esp8266/component.mk

@@ -8,11 +8,6 @@ FLASH_INIT_DATA			= $(SDK_BASE)/bin/esp_init_data_default.bin
 
 CUSTOM_TARGETS			+= $(FLASH_INIT_DATA)
 
-FLASH_INIT_CHUNKS += \
-	$(call FlashOffset,0x5000)=$(BLANK_BIN) \
-	$(call FlashOffset,0x4000)=$(FLASH_INIT_DATA) \
-	$(call FlashOffset,0x2000)=$(BLANK_BIN)
-
 # => 'Internal' SDK - for SDK Version 3+ as submodule in Sming repository
 # SDK_BASE just needs to point into our repo as it's overridden with the correct submodule path
 # This provides backward-compatiblity, so $(SMING)/third-party/ESP8266_NONOS_SDK) still works

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -14,6 +14,7 @@
 #include <driver/hw_timer.h>
 #include <gdb/gdb_hooks.h>
 #include <esp_cplusplus.h>
+#include <Storage.h>
 
 extern void init();
 
@@ -54,46 +55,9 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)
 
 extern "C" uint32 ICACHE_FLASH_ATTR  WEAK_ATTR user_rf_cal_sector_set(void)
 {
-    enum flash_size_map size_map = system_get_flash_size_map();
-    uint32 rf_cal_sec = 0;
-
-    switch (int(size_map)) {
-        case FLASH_SIZE_2M:
-            rf_cal_sec = 64 - 5;
-            break;
-
-        case FLASH_SIZE_4M_MAP_256_256:
-            rf_cal_sec = 128 - 5;
-            break;
-
-        case FLASH_SIZE_8M_MAP_512_512:
-            rf_cal_sec = 256 - 5;
-            break;
-
-        case FLASH_SIZE_16M_MAP_512_512:
-        case FLASH_SIZE_16M_MAP_1024_1024:
-            rf_cal_sec = 512 - 5;
-            break;
-
-        case FLASH_SIZE_32M_MAP_512_512:
-        case FLASH_SIZE_32M_MAP_1024_1024:
-            rf_cal_sec = 1024 - 5;
-            break;
-
-        case 8: // FLASH_SIZE_64M_MAP_1024_1024
-            rf_cal_sec = 2048 - 5;
-            break;
-
-        case 9: // FLASH_SIZE_128M_MAP_1024_1024
-            rf_cal_sec = 4096 - 5;
-            break;
-
-        default:
-            rf_cal_sec = 0;
-            break;
-    }
-
-    return rf_cal_sec;
+	// RF calibration stored in last sector of sysParam
+	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+	return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
 }
 
 #ifdef SDK_INTERNAL
@@ -104,25 +68,19 @@ extern "C" uint32 ICACHE_FLASH_ATTR  WEAK_ATTR user_rf_cal_sector_set(void)
 
 extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 {
-	const uint32_t MAX_PROGRAM_SECTORS = 0x100000 / SPI_FLASH_SEC_SIZE; // 1MB addressable
+	Storage::initialize();
 
-	// WARNING: Sming supports SDK 3.0 with rBoot enabled apps ONLY!
-	const partition_type_t SYSTEM_PARTITION_RBOOT_CONFIG = static_cast<partition_type_t>(SYSTEM_PARTITION_CUSTOMER_BEGIN + 0);
-	const partition_type_t SYSTEM_PARTITION_PROGRAM = static_cast<partition_type_t>(SYSTEM_PARTITION_CUSTOMER_BEGIN + 1);
+	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+	auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
 
-	// Partitions offsets and sizes must be in sector multiples, so work in sectors
-	#define PARTITION_ITEM(_type, _start, _length) \
-		{_type, (_start) * SPI_FLASH_SEC_SIZE, (_length) * SPI_FLASH_SEC_SIZE}
+	// RF calibration stored in last sector of sysParam
+	auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;
 
-	// Partitions in position order
-	uint32_t rfCalSector = user_rf_cal_sector_set();
 	static const partition_item_t partitions[] = {
-			PARTITION_ITEM(SYSTEM_PARTITION_BOOTLOADER,			0,					1),
-			PARTITION_ITEM(SYSTEM_PARTITION_RBOOT_CONFIG,		1,					1),
-			PARTITION_ITEM(SYSTEM_PARTITION_PROGRAM,			2,					std::min(MAX_PROGRAM_SECTORS, rfCalSector) - 2),
-			PARTITION_ITEM(SYSTEM_PARTITION_RF_CAL,				rfCalSector,		1),
-			PARTITION_ITEM(SYSTEM_PARTITION_PHY_DATA,			rfCalSector + 1,	1),
-			PARTITION_ITEM(SYSTEM_PARTITION_SYSTEM_PARAMETER,	rfCalSector + 2,	3),
+			{SYSTEM_PARTITION_BOOTLOADER,		0,						SPI_FLASH_SEC_SIZE},
+			{SYSTEM_PARTITION_PHY_DATA,			phy.address(),			phy.size()},
+			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParamSize},
+			{SYSTEM_PARTITION_RF_CAL,			sysParam.address() + sysParamSize,	SPI_FLASH_SEC_SIZE},
 	};
 
 	enum flash_size_map sizeMap = system_get_flash_size_map();
@@ -134,7 +92,7 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 			os_printf("partition[%u]: %u, 0x%08x, 0x%08x\n", i, part.type, part.addr, part.size);
 		}
 		if(sizeMap < FLASH_SIZE_8M_MAP_512_512) {
-			os_printf("** Note: SDK 3.0.1 requires SPI_SIZE >= 1M\n");
+			os_printf("** Note: SDK 3.0.1 requires spiFlash size >= 1M\n");
 		}
 		while(1) {
 			// Cannot proceed

Sming/Arch/Esp8266/Tools/ci/build.run.sh

@@ -1,7 +1,11 @@
 # Esp8266 build.run.sh
 
-make -C "$SMING_PROJECTS_DIR/samples/HttpServer_FirmwareUpload" python-requirements
 $MAKE_PARALLEL samples
 make clean samples-clean
 $MAKE_PARALLEL Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3
 $MAKE_PARALLEL HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1
+
+# Some samples (UPnP, for example) require more recent compiler
+if [ "$BUILD_COMPILER" == "eqt" ]; then
+    $MAKE_PARALLEL component-samples
+fi

Sming/Arch/Esp8266/app.mk

@@ -71,24 +71,7 @@ $(TARGET_OUT_1): $(COMPONENTS_AR) $(LIBMAIN_DST)
 
 ##@Flashing
 
-.PHONY: flashboot
-flashboot: $(RBOOT_BIN) ##Write just the rBoot boot sector
-	$(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS))
-
 .PHONY: flashconfig
 flashconfig: kill_term ##Erase the rBoot config sector
 	$(info Erasing rBoot config sector)
 	$(call WriteFlash,$(FLASH_RBOOT_ERASE_CONFIG_CHUNKS))
-
-.PHONY: flashapp
-flashapp: all kill_term ##Write just the application image
-	$(call WriteFlash,$(FLASH_RBOOT_APP_CHUNKS))
-
-.PHONY: flash
-flash: all kill_term ##Write the rBoot boot sector, application image and (if enabled) SPIFFS image
-	$(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS) $(FLASH_RBOOT_APP_CHUNKS) $(FLASH_SPIFFS_CHUNKS))
-ifeq ($(ENABLE_GDB), 1)
-	$(GDB_CMDLINE)
-else
-	$(TERMINAL)
-endif

Sming/Arch/Esp8266/spiffs-two-roms.hw

@@ -0,0 +1,13 @@
+{
+	"name": "Two ROM slots with single SPIFFS",
+	"base_config": "spiffs",
+	"partitions": {
+		"rom1": {
+			"address": "0x108000",
+			"size": "992K",
+			"type": "app",
+			"subtype": "ota_0",
+			"filename": "$(RBOOT_ROM_1_BIN)"
+		}
+	}
+}