From 89af8c740a2c9148e3e216e32193fe194f601039 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 3 Jan 2021 12:40:31 +0000 Subject: [PATCH 01/36] More CString methods --- Sming/Core/Data/CString.h | 62 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 59 insertions(+), 3 deletions(-) diff --git a/Sming/Core/Data/CString.h b/Sming/Core/Data/CString.h index 9dea4b4d9b..4335b95bc7 100644 --- a/Sming/Core/Data/CString.h +++ b/Sming/Core/Data/CString.h @@ -13,6 +13,7 @@ #pragma once #include +#include #include /** @@ -27,13 +28,21 @@ class CString : public std::unique_ptr public: CString() = default; - CString(const CString& src) = default; + CString(const CString& src) + { + assign(src.get()); + } CString(const String& src) { assign(src); } + CString(const char* src) + { + assign(src); + } + void assign(const String& src) { if(src) { @@ -59,6 +68,12 @@ class CString : public std::unique_ptr } } + CString& operator=(const CString& src) + { + assign(src.get()); + return *this; + } + CString& operator=(const String& src) { assign(src); @@ -81,14 +96,55 @@ class CString : public std::unique_ptr return get() ?: ""; } - bool operator==(const CString& other) const + bool equals(const CString& other) const { return strcmp(c_str(), other.c_str()) == 0; } + bool equals(const String& other) const + { + return other.equals(c_str()); + } + + bool equals(const char* other) const + { + if(other == nullptr) { + return length() == 0; + } + return strcmp(c_str(), other) == 0; + } + + bool equalsIgnoreCase(const CString& other) const + { + return strcasecmp(c_str(), other.c_str()) == 0; + } + + bool equalsIgnoreCase(const String& other) const + { + return other.equalsIgnoreCase(c_str()); + } + + bool equalsIgnoreCase(const char* other) const + { + if(other == nullptr) { + return length() == 0; + } + return strcasecmp(c_str(), other) == 0; + } + + bool operator==(const CString& other) const + { + return equals(other); + } + bool operator==(const String& other) const { - return strcmp(c_str(), other.c_str()) == 0; + return equals(other); + } + + bool operator==(const char* other) const + { + return equals(other); } size_t length() const From 418b99f1995c2b443582414ebe3312a659faac42 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Thu, 6 Feb 2020 23:35:52 +0000 Subject: [PATCH 02/36] Add gen_esp32part.py from ESP-IDF --- Sming/Components/Storage/Tools/README.rst | 25 + .../Components/Storage/Tools/gen_esp32part.py | 537 ++++++++++++++++++ 2 files changed, 562 insertions(+) create mode 100644 Sming/Components/Storage/Tools/README.rst create mode 100644 Sming/Components/Storage/Tools/gen_esp32part.py diff --git a/Sming/Components/Storage/Tools/README.rst b/Sming/Components/Storage/Tools/README.rst new file mode 100644 index 0000000000..549a58abbb --- /dev/null +++ b/Sming/Components/Storage/Tools/README.rst @@ -0,0 +1,25 @@ +Partition tools +=============== + +hwconfig.py + Supports parsing and creation of hardware configuration files. + + +Notes +----- + +Comparison between ESP-IDF and ESP8266-RTOS versions: + +gen_esp32part.py + Same name but slightly different: + ESP32 application partitions are aligned to 0x10000 boundary, ESP8266 to 0x1000 + + This is the ESP32 version. + Consider modifying, e.g add paramter to change alignment. + +parttool.py + From ESP-IDF. More recent, a few exception handling tweaks. + +gen_empty_partition.py + Identical. + diff --git a/Sming/Components/Storage/Tools/gen_esp32part.py b/Sming/Components/Storage/Tools/gen_esp32part.py new file mode 100644 index 0000000000..3406478cbe --- /dev/null +++ b/Sming/Components/Storage/Tools/gen_esp32part.py @@ -0,0 +1,537 @@ +#!/usr/bin/env python +# +# ESP32 partition table generation tool +# +# Converts partition tables to/from CSV and binary formats. +# +# See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html +# for explanation of partition table structure and uses. +# +# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import print_function, division +from __future__ import unicode_literals +import argparse +import os +import re +import struct +import sys +import hashlib +import binascii +import errno + +MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature +MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum +PARTITION_TABLE_SIZE = 0x1000 # Size of partition table + +MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 +NUM_PARTITION_SUBTYPE_APP_OTA = 16 + +__version__ = '1.2' + +APP_TYPE = 0x00 +DATA_TYPE = 0x01 + +TYPES = { + "app": APP_TYPE, + "data": DATA_TYPE, +} + +# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h +SUBTYPES = { + APP_TYPE: { + "factory": 0x00, + "test": 0x20, + }, + DATA_TYPE: { + "ota": 0x00, + "phy": 0x01, + "nvs": 0x02, + "coredump": 0x03, + "nvs_keys": 0x04, + "efuse": 0x05, + "esphttpd": 0x80, + "fat": 0x81, + "spiffs": 0x82, + }, +} + +quiet = False +md5sum = True +secure = False +offset_part_table = 0 + + +def status(msg): + """ Print status message to stderr """ + if not quiet: + critical(msg) + + +def critical(msg): + """ Print critical message to stderr """ + sys.stderr.write(msg) + sys.stderr.write('\n') + + +class PartitionTable(list): + def __init__(self): + super(PartitionTable, self).__init__(self) + + @classmethod + def from_csv(cls, csv_contents): + res = PartitionTable() + lines = csv_contents.splitlines() + + def expand_vars(f): + f = os.path.expandvars(f) + m = re.match(r'(? 1) + + # print sorted duplicate partitions by name + if len(duplicates) != 0: + print("A list of partitions that have the same name:") + for p in sorted(self, key=lambda x:x.name): + if len(duplicates.intersection([p.name])) != 0: + print("%s" % (p.to_csv())) + raise InputError("Partition names must be unique") + + # check for overlaps + last = None + for p in sorted(self, key=lambda x:x.offset): + if p.offset < offset_part_table + PARTITION_TABLE_SIZE: + raise InputError("Partition offset 0x%x is below 0x%x" % (p.offset, offset_part_table + PARTITION_TABLE_SIZE)) + if last is not None and p.offset < last.offset + last.size: + raise InputError("Partition at 0x%x overlaps 0x%x-0x%x" % (p.offset, last.offset, last.offset + last.size - 1)) + last = p + + def flash_size(self): + """ Return the size that partitions will occupy in flash + (ie the offset the last partition ends at) + """ + try: + last = sorted(self, reverse=True)[0] + except IndexError: + return 0 # empty table! + return last.offset + last.size + + @classmethod + def from_binary(cls, b): + md5 = hashlib.md5() + result = cls() + for o in range(0,len(b),32): + data = b[o:o + 32] + if len(data) != 32: + raise InputError("Partition table length must be a multiple of 32 bytes") + if data == b'\xFF' * 32: + return result # got end marker + if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]: # check only the magic number part + if data[16:] == md5.digest(): + continue # the next iteration will check for the end marker + else: + raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:]))) + else: + md5.update(data) + result.append(PartitionDefinition.from_binary(data)) + raise InputError("Partition table is missing an end-of-table marker") + + def to_binary(self): + result = b"".join(e.to_binary() for e in self) + if md5sum: + result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() + if len(result) >= MAX_PARTITION_LENGTH: + raise InputError("Binary partition table length (%d) longer than max" % len(result)) + result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing + return result + + def to_csv(self, simple_formatting=False): + rows = ["# ESP-IDF Partition Table", + "# Name, Type, SubType, Offset, Size, Flags"] + rows += [x.to_csv(simple_formatting) for x in self] + return "\n".join(rows) + "\n" + + +class PartitionDefinition(object): + MAGIC_BYTES = b"\xAA\x50" + + ALIGNMENT = { + APP_TYPE: 0x10000, + DATA_TYPE: 0x04, + } + + # dictionary maps flag name (as used in CSV flags list, property name) + # to bit set in flags words in binary format + FLAGS = { + "encrypted": 0 + } + + # add subtypes for the 16 OTA slot values ("ota_XX, etc.") + for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA): + SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot + + def __init__(self): + self.name = "" + self.type = None + self.subtype = None + self.offset = None + self.size = None + self.encrypted = False + + @classmethod + def from_csv(cls, line, line_no): + """ Parse a line from the CSV """ + line_w_defaults = line + ",,,," # lazy way to support default fields + fields = [f.strip() for f in line_w_defaults.split(",")] + + res = PartitionDefinition() + res.line_no = line_no + res.name = fields[0] + res.type = res.parse_type(fields[1]) + res.subtype = res.parse_subtype(fields[2]) + res.offset = res.parse_address(fields[3]) + res.size = res.parse_address(fields[4]) + if res.size is None: + raise InputError("Size field can't be empty") + + flags = fields[5].split(":") + for flag in flags: + if flag in cls.FLAGS: + setattr(res, flag, True) + elif len(flag) > 0: + raise InputError("CSV flag column contains unknown flag '%s'" % (flag)) + + return res + + def __eq__(self, other): + return self.name == other.name and self.type == other.type \ + and self.subtype == other.subtype and self.offset == other.offset \ + and self.size == other.size + + def __repr__(self): + def maybe_hex(x): + return "0x%x" % x if x is not None else "None" + return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, + maybe_hex(self.offset), maybe_hex(self.size)) + + def __str__(self): + return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1) + + def __cmp__(self, other): + return self.offset - other.offset + + def __lt__(self, other): + return self.offset < other.offset + + def __gt__(self, other): + return self.offset > other.offset + + def __le__(self, other): + return self.offset <= other.offset + + def __ge__(self, other): + return self.offset >= other.offset + + def parse_type(self, strval): + if strval == "": + raise InputError("Field 'type' can't be left empty.") + return parse_int(strval, TYPES) + + def parse_subtype(self, strval): + if strval == "": + return 0 # default + return parse_int(strval, SUBTYPES.get(self.type, {})) + + def parse_address(self, strval): + if strval == "": + return None # PartitionTable will fill in default + return parse_int(strval) + + def verify(self): + if self.type is None: + raise ValidationError(self, "Type field is not set") + if self.subtype is None: + raise ValidationError(self, "Subtype field is not set") + if self.offset is None: + raise ValidationError(self, "Offset field is not set") + align = self.ALIGNMENT.get(self.type, 4) + if self.offset % align: + raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.offset, align)) + if self.size % align and secure: + raise ValidationError(self, "Size 0x%x is not aligned to 0x%x" % (self.size, align)) + if self.size is None: + raise ValidationError(self, "Size field is not set") + + if self.name in TYPES and TYPES.get(self.name, "") != self.type: + critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's " + "type (0x%x). Mistake in partition table?" % (self.name, self.type)) + all_subtype_names = [] + for names in (t.keys() for t in SUBTYPES.values()): + all_subtype_names += names + if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype: + critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has " + "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype)) + + STRUCT_FORMAT = b"<2sBBLL16sL" + + @classmethod + def from_binary(cls, b): + if len(b) != 32: + raise InputError("Partition definition length must be exactly 32 bytes. Got %d bytes." % len(b)) + res = cls() + (magic, res.type, res.subtype, res.offset, + res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) + if b"\x00" in res.name: # strip null byte padding from name string + res.name = res.name[:res.name.index(b"\x00")] + res.name = res.name.decode() + if magic != cls.MAGIC_BYTES: + raise InputError("Invalid magic bytes (%r) for partition definition" % magic) + for flag,bit in cls.FLAGS.items(): + if flags & (1 << bit): + setattr(res, flag, True) + flags &= ~(1 << bit) + if flags != 0: + critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags) + return res + + def get_flags_list(self): + return [flag for flag in self.FLAGS.keys() if getattr(self, flag)] + + def to_binary(self): + flags = sum((1 << self.FLAGS[flag]) for flag in self.get_flags_list()) + return struct.pack(self.STRUCT_FORMAT, + self.MAGIC_BYTES, + self.type, self.subtype, + self.offset, self.size, + self.name.encode(), + flags) + + def to_csv(self, simple_formatting=False): + def addr_format(a, include_sizes): + if not simple_formatting and include_sizes: + for (val, suffix) in [(0x100000, "M"), (0x400, "K")]: + if a % val == 0: + return "%d%s" % (a // val, suffix) + return "0x%x" % a + + def lookup_keyword(t, keywords): + for k,v in keywords.items(): + if simple_formatting is False and t == v: + return k + return "%d" % t + + def generate_text_flags(): + """ colon-delimited list of flags """ + return ":".join(self.get_flags_list()) + + return ",".join([self.name, + lookup_keyword(self.type, TYPES), + lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})), + addr_format(self.offset, False), + addr_format(self.size, True), + generate_text_flags()]) + + +def parse_int(v, keywords={}): + """Generic parser for integer fields - int(x,0) with provision for + k/m/K/M suffixes and 'keyword' value lookup. + """ + try: + for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]: + if v.lower().endswith(letter): + return parse_int(v[:-1], keywords) * multiplier + return int(v, 0) + except ValueError: + if len(keywords) == 0: + raise InputError("Invalid field value %s" % v) + try: + return keywords[v.lower()] + except KeyError: + raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) + + +def main(): + global quiet + global md5sum + global offset_part_table + global secure + parser = argparse.ArgumentParser(description='ESP32 partition table utility') + + parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash', + nargs='?', choices=['1MB', '2MB', '4MB', '8MB', '16MB']) + parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true') + parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true') + parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is " + "enabled by default and this flag does nothing.", action='store_true') + parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true') + parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000') + parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true') + parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb')) + parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.', + nargs='?', default='-') + + args = parser.parse_args() + + quiet = args.quiet + md5sum = not args.disable_md5sum + secure = args.secure + offset_part_table = int(args.offset, 0) + input = args.input.read() + input_is_binary = input[0:2] == PartitionDefinition.MAGIC_BYTES + if input_is_binary: + status("Parsing binary partition input...") + table = PartitionTable.from_binary(input) + else: + input = input.decode() + status("Parsing CSV input...") + table = PartitionTable.from_csv(input) + + if not args.no_verify: + status("Verifying table...") + table.verify() + + if args.flash_size: + size_mb = int(args.flash_size.replace("MB", "")) + size = size_mb * 1024 * 1024 # flash memory uses honest megabytes! + table_size = table.flash_size() + if size < table_size: + raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured " + "flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu." % + (args.input.name, table_size / 1024.0 / 1024.0, table_size, size_mb)) + + # Make sure that the output directory is created + output_dir = os.path.abspath(os.path.dirname(args.output)) + + if not os.path.exists(output_dir): + try: + os.makedirs(output_dir) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + if input_is_binary: + output = table.to_csv() + with sys.stdout if args.output == '-' else open(args.output, 'w') as f: + f.write(output) + else: + output = table.to_binary() + try: + stdout_binary = sys.stdout.buffer # Python 3 + except AttributeError: + stdout_binary = sys.stdout + with stdout_binary if args.output == '-' else open(args.output, 'wb') as f: + f.write(output) + + +class InputError(RuntimeError): + def __init__(self, e): + super(InputError, self).__init__(e) + + +class ValidationError(InputError): + def __init__(self, partition, message): + super(ValidationError, self).__init__( + "Partition %s invalid: %s" % (partition.name, message)) + + +if __name__ == '__main__': + try: + main() + except InputError as e: + print(e, file=sys.stderr) + sys.exit(2) From 408adbd55e91864ff7d0fe67701cec3234bc28a4 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Thu, 3 Dec 2020 12:50:15 +0000 Subject: [PATCH 03/36] Fix gen_esp32part alignment handling Change app partitions minimum alignment to 1 sector (from 16) --- Sming/Components/Storage/Tools/gen_esp32part.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Sming/Components/Storage/Tools/gen_esp32part.py b/Sming/Components/Storage/Tools/gen_esp32part.py index 3406478cbe..9a28b39fc9 100644 --- a/Sming/Components/Storage/Tools/gen_esp32part.py +++ b/Sming/Components/Storage/Tools/gen_esp32part.py @@ -124,7 +124,7 @@ def expand_vars(f): raise InputError("CSV Error: Partitions overlap. Partition at line %d sets offset 0x%x. Previous partition ends 0x%x" % (e.line_no, e.offset, last_end)) if e.offset is None: - pad_to = 0x10000 if e.type == APP_TYPE else 4 + pad_to = e.alignment() if last_end % pad_to != 0: last_end += pad_to - (last_end % pad_to) e.offset = last_end @@ -251,7 +251,7 @@ class PartitionDefinition(object): MAGIC_BYTES = b"\xAA\x50" ALIGNMENT = { - APP_TYPE: 0x10000, + APP_TYPE: 0x1000, DATA_TYPE: 0x04, } @@ -342,6 +342,9 @@ def parse_address(self, strval): return None # PartitionTable will fill in default return parse_int(strval) + def alignment(self): + return self.ALIGNMENT.get(self.type, 4) + def verify(self): if self.type is None: raise ValidationError(self, "Type field is not set") @@ -349,7 +352,7 @@ def verify(self): raise ValidationError(self, "Subtype field is not set") if self.offset is None: raise ValidationError(self, "Offset field is not set") - align = self.ALIGNMENT.get(self.type, 4) + align = self.alignment() if self.offset % align: raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.offset, align)) if self.size % align and secure: @@ -491,7 +494,7 @@ def main(): table_size = table.flash_size() if size < table_size: raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured " - "flash size %dMB. Change the flash size in menuconfig under the 'Serial Flasher Config' menu." % + "flash size %dMB. Please change SPI_SIZE setting." % (args.input.name, table_size / 1024.0 / 1024.0, table_size, size_mb)) # Make sure that the output directory is created From e8f40e0cc17f5b2aad46271c0d32260827ad4ad9 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 4 Dec 2020 20:30:41 +0000 Subject: [PATCH 04/36] Rename gen_esp32part.py to hwconfig.py --- .../Tools/{gen_esp32part.py => hwconfig.py} | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) rename Sming/Components/Storage/Tools/{gen_esp32part.py => hwconfig.py} (96%) diff --git a/Sming/Components/Storage/Tools/gen_esp32part.py b/Sming/Components/Storage/Tools/hwconfig.py similarity index 96% rename from Sming/Components/Storage/Tools/gen_esp32part.py rename to Sming/Components/Storage/Tools/hwconfig.py index 9a28b39fc9..52504307a9 100644 --- a/Sming/Components/Storage/Tools/gen_esp32part.py +++ b/Sming/Components/Storage/Tools/hwconfig.py @@ -1,11 +1,10 @@ #!/usr/bin/env python # -# ESP32 partition table generation tool +# Sming hardware configuration tool # -# Converts partition tables to/from CSV and binary formats. +# Forked from Espressif gen_esp32part.py # -# See https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/partition-tables.html -# for explanation of partition table structure and uses. +# Original license: # # Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD # @@ -31,9 +30,9 @@ import binascii import errno -MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature +MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum -PARTITION_TABLE_SIZE = 0x1000 # Size of partition table +PARTITION_TABLE_SIZE = 0x1000 # Size of partition table MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 NUM_PARTITION_SUBTYPE_APP_OTA = 16 @@ -86,6 +85,7 @@ def critical(msg): class PartitionTable(list): + def __init__(self): super(PartitionTable, self).__init__(self) @@ -215,7 +215,7 @@ def flash_size(self): def from_binary(cls, b): md5 = hashlib.md5() result = cls() - for o in range(0,len(b),32): + for o in range(0, len(b), 32): data = b[o:o + 32] if len(data) != 32: raise InputError("Partition table length must be a multiple of 32 bytes") @@ -304,8 +304,10 @@ def __eq__(self, other): and self.size == other.size def __repr__(self): + def maybe_hex(x): return "0x%x" % x if x is not None else "None" + return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, maybe_hex(self.offset), maybe_hex(self.size)) @@ -384,7 +386,7 @@ def from_binary(cls, b): res.name = res.name.decode() if magic != cls.MAGIC_BYTES: raise InputError("Invalid magic bytes (%r) for partition definition" % magic) - for flag,bit in cls.FLAGS.items(): + for flag, bit in cls.FLAGS.items(): if flags & (1 << bit): setattr(res, flag, True) flags &= ~(1 << bit) @@ -405,6 +407,7 @@ def to_binary(self): flags) def to_csv(self, simple_formatting=False): + def addr_format(a, include_sizes): if not simple_formatting and include_sizes: for (val, suffix) in [(0x100000, "M"), (0x400, "K")]: @@ -413,7 +416,7 @@ def addr_format(a, include_sizes): return "0x%x" % a def lookup_keyword(t, keywords): - for k,v in keywords.items(): + for k, v in keywords.items(): if simple_formatting is False and t == v: return k return "%d" % t @@ -453,7 +456,7 @@ def main(): global md5sum global offset_part_table global secure - parser = argparse.ArgumentParser(description='ESP32 partition table utility') + parser = argparse.ArgumentParser(description='Sming hardware configuration utility') parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash', nargs='?', choices=['1MB', '2MB', '4MB', '8MB', '16MB']) @@ -494,7 +497,7 @@ def main(): table_size = table.flash_size() if size < table_size: raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured " - "flash size %dMB. Please change SPI_SIZE setting." % + "flash size %dMB. Please change SPI_SIZE setting." % (args.input.name, table_size / 1024.0 / 1024.0, table_size, size_mb)) # Make sure that the output directory is created @@ -522,11 +525,13 @@ def main(): class InputError(RuntimeError): + def __init__(self, e): super(InputError, self).__init__(e) class ValidationError(InputError): + def __init__(self, partition, message): super(ValidationError, self).__init__( "Partition %s invalid: %s" % (partition.name, message)) From 6c8e95b658314f6e3d7291df65abfacd6320b5b7 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Wed, 2 Dec 2020 22:10:04 +0000 Subject: [PATCH 05/36] Implement JSON parsing and construction Add 'create-config' target Create default hardware config if not present Generate partition table from JSON --- .../Components/esp32/sdk/partitions/base.csv | 5 - Sming/Components/Storage/Tools/hwconfig.py | 184 +++++++++++++----- 2 files changed, 134 insertions(+), 55 deletions(-) delete mode 100644 Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv diff --git a/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv b/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv deleted file mode 100644 index 97c8e0148e..0000000000 --- a/Sming/Arch/Esp32/Components/esp32/sdk/partitions/base.csv +++ /dev/null @@ -1,5 +0,0 @@ -# Name, Type, SubType, Offset, Size, Flags -# Note: if you have increased the bootloader size, make sure to update the offsets to avoid overlap -nvs, data, nvs, 0x9000, 0x6000, -phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 0x1F0000, diff --git a/Sming/Components/Storage/Tools/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig.py index 52504307a9..1a0f355907 100644 --- a/Sming/Components/Storage/Tools/hwconfig.py +++ b/Sming/Components/Storage/Tools/hwconfig.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Sming hardware configuration tool # @@ -23,12 +23,12 @@ from __future__ import unicode_literals import argparse import os -import re import struct import sys import hashlib import binascii import errno +import json, configparser MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum @@ -90,27 +90,22 @@ def __init__(self): super(PartitionTable, self).__init__(self) @classmethod - def from_csv(cls, csv_contents): + def from_json(cls, table): res = PartitionTable() - lines = csv_contents.splitlines() - - def expand_vars(f): - f = os.path.expandvars(f) - m = re.match(r'(? 0: - raise InputError("CSV flag column contains unknown flag '%s'" % (flag)) + res.line_no = index + res.name = label + res.type = res.parse_type(entry['type']) + res.subtype = res.parse_subtype(entry['subtype']) + res.offset = res.parse_address(entry['address']) + res.size = res.parse_address(entry['size']) + if res.offset is None or res.size is None: + raise InputError("Offset/Size fields may not be empty") + + if 'flags' in entry: + flags = entry['flags'].split(":") + for flag in flags: + if flag in cls.FLAGS: + setattr(res, flag, True) + elif len(flag) > 0: + raise InputError("Unknown flag '%s' in partition entry '%s'" % (flag, label)) return res @@ -451,6 +444,90 @@ def parse_int(v, keywords={}): raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) +def createConfig(input, output): + """Parse makefile variables to create default configuration""" + + print("createConfig(%s, %s)" % (input.name, output)) + parser = configparser.ConfigParser() + parser.optionxform = str # Preserve case + data = "[DEFAULT]\r\n" + input.read().decode() + parser.read_string(data) + vars = parser['DEFAULT'] + + config = {} + config['name'] ='Generated configuration' + arch = os.environ['SMING_ARCH'] + config['arch'] = arch + if arch == 'Esp8266': + config['flash-size'] = vars['SPI_SIZE'] + config['spi-mode'] = vars['SPI_MODE'] + config['spi-speed'] = vars['SPI_SPEED'] + table = {} + config['partition-table'] = table + table['offset'] = "0x2000" + entries = {} + table['entries'] = entries + + def createEntry(address, size, type, subtype): + entry = {} + entry['address'] = '0x%06x' % address + entry['size'] = '0x%06x' % size + entry['type'] = type + entry['subtype'] = subtype + return entry + + def tryAddEntry(label, addressVar, type, subtype): + if not addressVar in vars: + return + s = vars[addressVar] + if s == '': + return None + addr = parse_int(s) + size= 0x100000 - (addr % 0x100000) + entry = createEntry(addr, size, type, subtype) + entries[label] = entry + return entry + + def addMake(entry, target): + make = {} + make['target'] = target + entry['make'] = make + return make + + def firmwareFilename(name, ext = '.bin'): + return os.environ['BUILD_BASE'] + '/' + vars[name] + ext + + if arch == 'Esp32': + entries['nvs'] = createEntry(0x9000, 0x6000, "data", "nvs") + entries['phy_init'] = createEntry(0xf000, 0x1000, "data", "phy") + entries['factory'] = createEntry(0x10000, 0x1f000, "app", "factory") + vars['RBOOT_SPIFFS_0'] = '0x100000' + else: + entries['phy_init'] = createEntry(0x3000, 0x1000, "data", "phy") + entries['sysconfig'] = createEntry(0x4000, 0x4000, "data", "nvs") + entry = tryAddEntry('rom0', 'RBOOT_ROM0_ADDR', 'app', 'factory') + if not entry is None: + entry['filename'] = firmwareFilename('RBOOT_ROM_0') + addMake(entry, 'fwimage') + entry = tryAddEntry('rom1', 'RBOOT_ROM1_ADDR', 'app', 'ota_0') + if not entry is None: + entry['filename'] = firmwareFilename('RBOOT_ROM_1') + addMake(entry, 'fwimage') + tryAddEntry('rom2', 'RBOOT_ROM2_ADDR', 'app', 'ota_1') + + entry = tryAddEntry('spiffs1', 'RBOOT_SPIFFS_0', 'data', 'spiffs') + if vars['DISABLE_SPIFFS'] == '0' and not entry is None: + entry['filename'] = firmwareFilename('SPIFF_BIN') + make = addMake(entry, 'spiffsgen') + make['content'] = vars['SPIFF_FILES'] + make['size'] = vars['SPIFF_SIZE'] + tryAddEntry('spiffs2', 'RBOOT_SPIFFS_1', 'data', 'spiffs') + + with sys.stdout if output == '-' else open(output, 'w') as f: + f.write(json.dumps(config, indent=4)) + + + def main(): global quiet global md5sum @@ -467,7 +544,7 @@ def main(): parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true') parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000') parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true') - parser.add_argument('input', help='Path to CSV or binary file to parse.', type=argparse.FileType('rb')) + parser.add_argument('input', help='Path to JSON or binary file to parse.', type=argparse.FileType('rb')) parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.', nargs='?', default='-') @@ -477,15 +554,22 @@ def main(): md5sum = not args.disable_md5sum secure = args.secure offset_part_table = int(args.offset, 0) + + if args.input.name.endswith('.mk'): + createConfig(args.input, args.output) + return + input = args.input.read() - input_is_binary = input[0:2] == PartitionDefinition.MAGIC_BYTES - if input_is_binary: + input_is_binary = (input[0:2] == PartitionDefinition.MAGIC_BYTES) + if args.input.name.endswith('.hw'): + status("Parsing JSON input...") + config = json.loads(input.decode()) + table = PartitionTable.from_json(config['partition-table']) + elif input_is_binary: status("Parsing binary partition input...") table = PartitionTable.from_binary(input) else: - input = input.decode() - status("Parsing CSV input...") - table = PartitionTable.from_csv(input) + raise InputError("Unknown input file format") if not args.no_verify: status("Verifying table...") From 61342fa152ab8731a91d8ecc3c66fccf597ed66b Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 5 Dec 2020 07:01:08 +0000 Subject: [PATCH 06/36] Move hwconfig into folder --- Sming/Components/Storage/Tools/{ => hwconfig}/hwconfig.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Sming/Components/Storage/Tools/{ => hwconfig}/hwconfig.py (100%) diff --git a/Sming/Components/Storage/Tools/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py similarity index 100% rename from Sming/Components/Storage/Tools/hwconfig.py rename to Sming/Components/Storage/Tools/hwconfig/hwconfig.py From 1bd8b734e380d46e8c17add3e9c1d3501802febd Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 5 Dec 2020 08:28:42 +0000 Subject: [PATCH 07/36] Split hwconfig into separate units, refactor/revise Fix warnings Fix size checking Rename 'offset' -> 'address', 'label' -> 'name' for consistency Add explicit command field to hwconfig tool List OTA subtypes explicitly, specify alignments by arch Add memory map generator PartitionTable entries sorted by default, simplify Map (base on Table) Add standard configs and support configuration inheritance via 'base_config' value Pull PARTITION_OFFSET from hardware config Add data/sysparam type Support external devices via Storage::Device Generate binary partition table with 'SMING_EXTENSIONS' pseudo-entry which esp32 bootloader accepts as end of table md5sum always enabled Support 'G' in size fields Add partition read-only flag Add `config.buildVars()` method to generate list of build variables Validate config against schema Enforce no-whitespace in names Fix quiet/secure settings Add user type, size_bytes --- Sming/Arch/Esp32/standard.hw | 38 + Sming/Arch/Esp8266/spiffs-two-roms.hw | 14 + Sming/Arch/Esp8266/standard.hw | 39 ++ Sming/Arch/Esp8266/two-rom-mode.hw | 16 + Sming/Arch/Host/standard.hw | 21 + Sming/Components/Storage/Tools/README.rst | 21 +- .../Storage/Tools/hwconfig/common.py | 94 +++ .../Storage/Tools/hwconfig/config.py | 104 +++ .../Storage/Tools/hwconfig/hwconfig.py | 659 ++---------------- .../Storage/Tools/hwconfig/partition.py | 538 ++++++++++++++ .../Storage/Tools/hwconfig/storage.py | 116 +++ .../Components/{esptool => Storage}/blank.bin | 0 Sming/Components/Storage/requirements.txt | 1 + Sming/Components/Storage/schema.json | 172 +++++ Sming/Components/spiffs/blankfs.bin | 1 - Sming/spiffs.hw | 18 + Sming/standard-4m.hw | 9 + 17 files changed, 1241 insertions(+), 620 deletions(-) create mode 100644 Sming/Arch/Esp32/standard.hw create mode 100644 Sming/Arch/Esp8266/spiffs-two-roms.hw create mode 100644 Sming/Arch/Esp8266/standard.hw create mode 100644 Sming/Arch/Esp8266/two-rom-mode.hw create mode 100644 Sming/Arch/Host/standard.hw create mode 100644 Sming/Components/Storage/Tools/hwconfig/common.py create mode 100644 Sming/Components/Storage/Tools/hwconfig/config.py create mode 100644 Sming/Components/Storage/Tools/hwconfig/partition.py create mode 100644 Sming/Components/Storage/Tools/hwconfig/storage.py rename Sming/Components/{esptool => Storage}/blank.bin (100%) create mode 100644 Sming/Components/Storage/requirements.txt create mode 100644 Sming/Components/Storage/schema.json delete mode 100644 Sming/Components/spiffs/blankfs.bin create mode 100644 Sming/spiffs.hw create mode 100644 Sming/standard-4m.hw diff --git a/Sming/Arch/Esp32/standard.hw b/Sming/Arch/Esp32/standard.hw new file mode 100644 index 0000000000..4e9931ff28 --- /dev/null +++ b/Sming/Arch/Esp32/standard.hw @@ -0,0 +1,38 @@ +{ + "name": "Standard config with single ROM", + "comment": "Should work with any Esp32 variant", + "arch": "Esp32", + "partition_table_offset": "0x8000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "4M", + "mode": "dio", + "speed": 40 + } + }, + "partitions": { + "phy_init": { + "address": "0x00f000", + "size": "0x1000", + "type": "data", + "subtype": "phy", + "flags": "" + }, + "nvs": { + "address": "0x009000", + "size": "0x6000", + "type": "data", + "subtype": "nvs", + "flags": "" + }, + "factory": { + "address": "0x010000", + "size": "0x1f0000", + "type": "app", + "subtype": "factory", + "flags": "", + "filename": "$(TARGET_BIN)" + } + } +} diff --git a/Sming/Arch/Esp8266/spiffs-two-roms.hw b/Sming/Arch/Esp8266/spiffs-two-roms.hw new file mode 100644 index 0000000000..897c826074 --- /dev/null +++ b/Sming/Arch/Esp8266/spiffs-two-roms.hw @@ -0,0 +1,14 @@ +{ + "name": "Two ROM slots with single SPIFFS", + "base_config": "spiffs", + "partitions": { + "rom1": { + "address": "0x108000", + "size": "992K", + "type": "app", + "subtype": "ota_0", + "flags": "", + "filename": "$(RBOOT_ROM_1_BIN)" + } + } +} diff --git a/Sming/Arch/Esp8266/standard.hw b/Sming/Arch/Esp8266/standard.hw new file mode 100644 index 0000000000..b2f45be55f --- /dev/null +++ b/Sming/Arch/Esp8266/standard.hw @@ -0,0 +1,39 @@ +{ + "name": "Standard config with single ROM", + "comment": "Should work with any Esp8266 variant", + "arch": "Esp8266", + "partition_table_offset": "0x2000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "1M", + "mode": "dio", + "speed": 40 + } + }, + "partitions": { + "phy_init": { + "address": "0x003000", + "size": "4K", + "type": "data", + "subtype": "phy", + "flags": "", + "filename": "$(FLASH_INIT_DATA)" + }, + "sys_param": { + "address": "0x004000", + "size": "16K", + "type": "data", + "subtype": "sysparam", + "flags": "" + }, + "rom0": { + "address": "0x008000", + "size": "992K", + "type": "app", + "subtype": "factory", + "flags": "", + "filename": "$(RBOOT_ROM_0_BIN)" + } + } +} diff --git a/Sming/Arch/Esp8266/two-rom-mode.hw b/Sming/Arch/Esp8266/two-rom-mode.hw new file mode 100644 index 0000000000..9d6399179f --- /dev/null +++ b/Sming/Arch/Esp8266/two-rom-mode.hw @@ -0,0 +1,16 @@ +{ + "name": "Two ROM slots in 1MB of flash", + "base_config": "standard", + "partitions": { + "rom0": { + "size": "480K" + }, + "rom1": { + "address": "0x80000", + "size": "512K", + "type": "app", + "subtype": "ota_0", + "filename": "$(RBOOT_ROM_1_BIN)" + } + } +} diff --git a/Sming/Arch/Host/standard.hw b/Sming/Arch/Host/standard.hw new file mode 100644 index 0000000000..119ddd8ba8 --- /dev/null +++ b/Sming/Arch/Host/standard.hw @@ -0,0 +1,21 @@ +{ + "name": "Standard config with single ROM", + "arch": "Host", + "partition_table_offset": "0x2000", + "devices": { + "spiFlash": { + "type": "flash", + "size": "4M" + } + }, + "partitions": { + "rom0": { + "address": "0x008000", + "size": "992K", + "type": "app", + "subtype": "factory", + "flags": "", + "filename": "$(BLANK_BIN)" + } + } +} diff --git a/Sming/Components/Storage/Tools/README.rst b/Sming/Components/Storage/Tools/README.rst index 549a58abbb..04d56eacff 100644 --- a/Sming/Components/Storage/Tools/README.rst +++ b/Sming/Components/Storage/Tools/README.rst @@ -2,24 +2,5 @@ Partition tools =============== hwconfig.py - Supports parsing and creation of hardware configuration files. - - -Notes ------ - -Comparison between ESP-IDF and ESP8266-RTOS versions: - -gen_esp32part.py - Same name but slightly different: - ESP32 application partitions are aligned to 0x10000 boundary, ESP8266 to 0x1000 - - This is the ESP32 version. - Consider modifying, e.g add paramter to change alignment. - -parttool.py - From ESP-IDF. More recent, a few exception handling tweaks. - -gen_empty_partition.py - Identical. + Supports parsing and creation of hardware configuration files which include partition information. diff --git a/Sming/Components/Storage/Tools/hwconfig/common.py b/Sming/Components/Storage/Tools/hwconfig/common.py new file mode 100644 index 0000000000..8376eae740 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/common.py @@ -0,0 +1,94 @@ +# +# Common functions and definitions +# + +import sys, json, platform + +quiet = False + +def status(msg): + """ Print status message to stderr """ + if not quiet: + critical(msg) + + +def critical(msg): + """ Print critical message to stderr """ + sys.stderr.write(msg) + sys.stderr.write('\n') + + +def fixpath(path): + """ Paths in Windows can get a little weird """ + if len(path) > 2 and path[1] != ':' and platform.system() == 'Windows' and path[2] == '/': + return path[1] + ':' + path[2:] + return path + + +def parse_int(v, keywords=None): + """Generic parser for integer fields - int(x,0) with provision for + k/m/K/M suffixes and 'keyword' value lookup. + """ + try: + for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]: + if v.lower().endswith(letter): + return parse_int(v[:-1], keywords) * multiplier + return int(v, 0) + except ValueError: + if keywords is None: + raise InputError("Invalid field value %s" % v) + try: + return keywords[v.lower()] + except KeyError: + raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) + + +def stringnum(s): + """Return number if s contains only digits, otherwise return the string + """ + return int(s) if s.isdigit() else s + + +def addr_format(a): + return "0x%08x" % a + + +def size_format(a): + if a != 0: + for (val, suffix) in [(0x40000000, "G"), (0x100000, "M"), (0x400, "K")]: + if a % val == 0: + return "%d%s" % (a // val, suffix) + return "0x%08x" % a + + +def quote(v): + return '"' + v + '"' + + +def contains_whitespace(s): + return ''.join(s.split()) != s + + +def to_json(obj): + return json.dumps(obj, indent=4) + + +def lookup_keyword(t, keywords): + for k, v in keywords.items(): + if t == v: + return k + return "%d" % t + + +class InputError(RuntimeError): + + def __init__(self, e): + super(InputError, self).__init__(e) + + +class ValidationError(InputError): + + def __init__(self, partition, message): + super(ValidationError, self).__init__( + "Partition %s invalid: %s" % (partition.name, message)) + diff --git a/Sming/Components/Storage/Tools/hwconfig/config.py b/Sming/Components/Storage/Tools/hwconfig/config.py new file mode 100644 index 0000000000..db42da7bb2 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/config.py @@ -0,0 +1,104 @@ +# +# Configuration object +# + +import os, partition, storage +from common import * +from builtins import classmethod + +def findConfig(name): + dirs = os.environ['HWCONFIG_DIRS'].split(' ') + for d in dirs: + path = os.path.join(fixpath(d), name + '.hw') + if os.path.exists(path): + return path + raise InputError("Config '%s' not found" % name) + +class Config(object): + def __init__(self): + self.partitions = partition.Table() + self.devices = storage.List() + self.depends = [] + + def __str__(self): + return "'%s' for %s" % (self.name, self.arch) + + @classmethod + def from_name(cls, name): + config = Config() + config.load(name) + return config + + def load(self, name): + filename = findConfig(name) + self.depends.append(filename) + data = json.load(open(filename)) + self.parse_dict(data) + + def parse_dict(self, data): + base_config = data.get('base_config') + if base_config is not None: + self.load(base_config) + del data['base_config'] + + for k, v in data.items(): + if k == 'name': + self.name = v + elif k == 'arch': + self.arch = v + elif k == 'partition_table_offset': + self.partitions.offset = parse_int(v) + elif k == 'devices': + self.devices.parse_dict(v) + elif k == 'comment': + self.comment = v + elif k != 'partitions': + raise InputError("Unknown config key '%s'" % k) + + v = data.get('partitions') + if not v is None: + self.partitions.parse_dict(v, self.devices) + + def dict(self): + res = {} + + res['name'] = self.name + if hasattr(self, 'comment'): + res['comment'] = self.comment + res['arch'] = self.arch; + res['partition_table_offset'] = self.partitions.offset_str() + res['devices'] = self.devices.dict() + res['partitions'] = self.partitions.dict() + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + dict = {} + + dict['PARTITION_TABLE_OFFSET'] = self.partitions.offset_str() + + dict['HWCONFIG_DEPENDS'] = " ".join(self.depends) + dict.update(self.devices.buildVars()) + dict.update(self.partitions.buildVars()) + + res = "# Generated from hardware configuration '%s'\r\n" % self.name + for k, v in dict.items(): + res += "%s = %s\r\n" % (k, v) + return res + + def verify(self, secure): + self.partitions.verify(self.arch, secure) + + def map(self): + return partition.Map(self.partitions, self.devices) + + @classmethod + def from_binary(cls, b): + res = Config() + res.name = 'from binary' + res.arch = os.environ.get('SMING_ARCH', 'Unknown') + res.partitions.offset = 0 + res.partitions.parse_binary(b, res.devices) + return res diff --git a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py index 1a0f355907..527e5c824c 100644 --- a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py +++ b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py @@ -2,623 +2,84 @@ # # Sming hardware configuration tool # -# Forked from Espressif gen_esp32part.py -# -# Original license: -# -# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http:#www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import print_function, division -from __future__ import unicode_literals -import argparse -import os -import struct -import sys -import hashlib -import binascii -import errno -import json, configparser - -MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature -MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum -PARTITION_TABLE_SIZE = 0x1000 # Size of partition table - -MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 -NUM_PARTITION_SUBTYPE_APP_OTA = 16 - -__version__ = '1.2' - -APP_TYPE = 0x00 -DATA_TYPE = 0x01 - -TYPES = { - "app": APP_TYPE, - "data": DATA_TYPE, -} - -# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h -SUBTYPES = { - APP_TYPE: { - "factory": 0x00, - "test": 0x20, - }, - DATA_TYPE: { - "ota": 0x00, - "phy": 0x01, - "nvs": 0x02, - "coredump": 0x03, - "nvs_keys": 0x04, - "efuse": 0x05, - "esphttpd": 0x80, - "fat": 0x81, - "spiffs": 0x82, - }, -} - -quiet = False -md5sum = True -secure = False -offset_part_table = 0 - - -def status(msg): - """ Print status message to stderr """ - if not quiet: - critical(msg) +import common, argparse, os, partition +from common import * +from config import Config -def critical(msg): - """ Print critical message to stderr """ - sys.stderr.write(msg) - sys.stderr.write('\n') - - -class PartitionTable(list): - - def __init__(self): - super(PartitionTable, self).__init__(self) - - @classmethod - def from_json(cls, table): - res = PartitionTable() - - if 'offset' in table: - global offset_part_table - offset_part_table = parse_int(table['offset']) - - index = 0 - for (label, entry) in table['entries'].items(): - try: - res.append(PartitionDefinition.from_json(label, entry, index)) - index += 1 - except InputError as e: - raise InputError("Error in partition '%s'" % (label)) - except Exception: - critical("Unexpected error in partition '%s': %s" % (label, entry)) - raise - - # fix up missing offsets & negative sizes - last_end = offset_part_table + PARTITION_TABLE_SIZE # first offset after partition table - for e in res: - if e.offset is not None and e.offset < last_end: - if e == res[0]: - raise InputError("Error: First partition offset 0x%x overlaps end of partition table 0x%x" - % (e.offset, last_end)) - else: - raise InputError("Error: Partitions overlap. Partition '%s' sets offset 0x%x. Previous partition ends 0x%x" - % (e.name, e.offset, last_end)) - if e.offset is None: - pad_to = e.alignment() - if last_end % pad_to != 0: - last_end += pad_to - (last_end % pad_to) - e.offset = last_end - if e.size < 0: - e.size = -e.size - e.offset - last_end = e.offset + e.size - - return res - - def __getitem__(self, item): - """ Allow partition table access via name as well as by - numeric index. """ - if isinstance(item, str): - for x in self: - if x.name == item: - return x - raise ValueError("No partition entry named '%s'" % item) - else: - return super(PartitionTable, self).__getitem__(item) - - def find_by_type(self, ptype, subtype): - """ Return a partition by type & subtype, returns - None if not found """ - # convert ptype & subtypes names (if supplied this way) to integer values - try: - ptype = TYPES[ptype] - except KeyError: - try: - ptype = int(ptype, 0) - except TypeError: - pass +def openOutput(path): + if path == '-': try: - subtype = SUBTYPES[int(ptype)][subtype] - except KeyError: - try: - subtype = int(subtype, 0) - except TypeError: - pass - - for p in self: - if p.type == ptype and p.subtype == subtype: - yield p - return - - def find_by_name(self, name): - for p in self: - if p.name == name: - return p - return None - - def verify(self): - # verify each partition individually - for p in self: - p.verify() - - # check on duplicate name - names = [p.name for p in self] - duplicates = set(n for n in names if names.count(n) > 1) - - # print sorted duplicate partitions by name - if len(duplicates) != 0: - print("A list of partitions that have the same name:") - for p in sorted(self, key=lambda x:x.name): - if len(duplicates.intersection([p.name])) != 0: - print("%s" % (p.to_csv())) - raise InputError("Partition names must be unique") - - # check for overlaps - last = None - for p in sorted(self, key=lambda x:x.offset): - if p.offset < offset_part_table + PARTITION_TABLE_SIZE: - raise InputError("Partition '%s' offset 0x%x is below 0x%x" % (p.name, p.offset, offset_part_table + PARTITION_TABLE_SIZE)) - if last is not None and p.offset < last.offset + last.size: - raise InputError("Partition '%s' at 0x%x overlaps 0x%x-0x%x" % (p.name, p.offset, last.offset, last.offset + last.size - 1)) - last = p - - def flash_size(self): - """ Return the size that partitions will occupy in flash - (ie the offset the last partition ends at) - """ - try: - last = sorted(self, reverse=True)[0] - except IndexError: - return 0 # empty table! - return last.offset + last.size - - @classmethod - def from_binary(cls, b): - md5 = hashlib.md5() - result = cls() - for o in range(0, len(b), 32): - data = b[o:o + 32] - if len(data) != 32: - raise InputError("Partition table length must be a multiple of 32 bytes") - if data == b'\xFF' * 32: - return result # got end marker - if md5sum and data[:2] == MD5_PARTITION_BEGIN[:2]: # check only the magic number part - if data[16:] == md5.digest(): - continue # the next iteration will check for the end marker - else: - raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:]))) - else: - md5.update(data) - result.append(PartitionDefinition.from_binary(data)) - raise InputError("Partition table is missing an end-of-table marker") - - def to_binary(self): - result = b"".join(e.to_binary() for e in self) - if md5sum: - result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() - if len(result) >= MAX_PARTITION_LENGTH: - raise InputError("Binary partition table length (%d) longer than max" % len(result)) - result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing - return result - - def to_csv(self, simple_formatting=False): - rows = ["# ESP-IDF Partition Table", - "# Name, Type, SubType, Offset, Size, Flags"] - rows += [x.to_csv(simple_formatting) for x in self] - return "\n".join(rows) + "\n" - - -class PartitionDefinition(object): - MAGIC_BYTES = b"\xAA\x50" - - ALIGNMENT = { - APP_TYPE: 0x1000, - DATA_TYPE: 0x04, - } - - # dictionary maps flag name (as used in CSV flags list, property name) - # to bit set in flags words in binary format - FLAGS = { - "encrypted": 0 - } - - # add subtypes for the 16 OTA slot values ("ota_XX, etc.") - for ota_slot in range(NUM_PARTITION_SUBTYPE_APP_OTA): - SUBTYPES[TYPES["app"]]["ota_%d" % ota_slot] = MIN_PARTITION_SUBTYPE_APP_OTA + ota_slot - - def __init__(self): - self.name = "" - self.type = None - self.subtype = None - self.offset = None - self.size = None - self.encrypted = False - - @classmethod - def from_json(cls, label, entry, index): - """ Parse a JSON partition table entry """ - res = PartitionDefinition() - res.line_no = index - res.name = label - res.type = res.parse_type(entry['type']) - res.subtype = res.parse_subtype(entry['subtype']) - res.offset = res.parse_address(entry['address']) - res.size = res.parse_address(entry['size']) - if res.offset is None or res.size is None: - raise InputError("Offset/Size fields may not be empty") - - if 'flags' in entry: - flags = entry['flags'].split(":") - for flag in flags: - if flag in cls.FLAGS: - setattr(res, flag, True) - elif len(flag) > 0: - raise InputError("Unknown flag '%s' in partition entry '%s'" % (flag, label)) - - return res - - def __eq__(self, other): - return self.name == other.name and self.type == other.type \ - and self.subtype == other.subtype and self.offset == other.offset \ - and self.size == other.size - - def __repr__(self): - - def maybe_hex(x): - return "0x%x" % x if x is not None else "None" - - return "PartitionDefinition('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, - maybe_hex(self.offset), maybe_hex(self.size)) - - def __str__(self): - return "Part '%s' %d/%d @ 0x%x size 0x%x" % (self.name, self.type, self.subtype, self.offset or -1, self.size or -1) - - def __cmp__(self, other): - return self.offset - other.offset - - def __lt__(self, other): - return self.offset < other.offset - - def __gt__(self, other): - return self.offset > other.offset - - def __le__(self, other): - return self.offset <= other.offset - - def __ge__(self, other): - return self.offset >= other.offset - - def parse_type(self, strval): - if strval == "": - raise InputError("Field 'type' can't be left empty.") - return parse_int(strval, TYPES) - - def parse_subtype(self, strval): - if strval == "": - return 0 # default - return parse_int(strval, SUBTYPES.get(self.type, {})) - - def parse_address(self, strval): - if strval == "": - return None # PartitionTable will fill in default - return parse_int(strval) - - def alignment(self): - return self.ALIGNMENT.get(self.type, 4) - - def verify(self): - if self.type is None: - raise ValidationError(self, "Type field is not set") - if self.subtype is None: - raise ValidationError(self, "Subtype field is not set") - if self.offset is None: - raise ValidationError(self, "Offset field is not set") - align = self.alignment() - if self.offset % align: - raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.offset, align)) - if self.size % align and secure: - raise ValidationError(self, "Size 0x%x is not aligned to 0x%x" % (self.size, align)) - if self.size is None: - raise ValidationError(self, "Size field is not set") - - if self.name in TYPES and TYPES.get(self.name, "") != self.type: - critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's " - "type (0x%x). Mistake in partition table?" % (self.name, self.type)) - all_subtype_names = [] - for names in (t.keys() for t in SUBTYPES.values()): - all_subtype_names += names - if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype: - critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has " - "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype)) - - STRUCT_FORMAT = b"<2sBBLL16sL" - - @classmethod - def from_binary(cls, b): - if len(b) != 32: - raise InputError("Partition definition length must be exactly 32 bytes. Got %d bytes." % len(b)) - res = cls() - (magic, res.type, res.subtype, res.offset, - res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) - if b"\x00" in res.name: # strip null byte padding from name string - res.name = res.name[:res.name.index(b"\x00")] - res.name = res.name.decode() - if magic != cls.MAGIC_BYTES: - raise InputError("Invalid magic bytes (%r) for partition definition" % magic) - for flag, bit in cls.FLAGS.items(): - if flags & (1 << bit): - setattr(res, flag, True) - flags &= ~(1 << bit) - if flags != 0: - critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags) - return res - - def get_flags_list(self): - return [flag for flag in self.FLAGS.keys() if getattr(self, flag)] - - def to_binary(self): - flags = sum((1 << self.FLAGS[flag]) for flag in self.get_flags_list()) - return struct.pack(self.STRUCT_FORMAT, - self.MAGIC_BYTES, - self.type, self.subtype, - self.offset, self.size, - self.name.encode(), - flags) - - def to_csv(self, simple_formatting=False): - - def addr_format(a, include_sizes): - if not simple_formatting and include_sizes: - for (val, suffix) in [(0x100000, "M"), (0x400, "K")]: - if a % val == 0: - return "%d%s" % (a // val, suffix) - return "0x%x" % a - - def lookup_keyword(t, keywords): - for k, v in keywords.items(): - if simple_formatting is False and t == v: - return k - return "%d" % t - - def generate_text_flags(): - """ colon-delimited list of flags """ - return ":".join(self.get_flags_list()) - - return ",".join([self.name, - lookup_keyword(self.type, TYPES), - lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})), - addr_format(self.offset, False), - addr_format(self.size, True), - generate_text_flags()]) - - -def parse_int(v, keywords={}): - """Generic parser for integer fields - int(x,0) with provision for - k/m/K/M suffixes and 'keyword' value lookup. - """ - try: - for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]: - if v.lower().endswith(letter): - return parse_int(v[:-1], keywords) * multiplier - return int(v, 0) - except ValueError: - if len(keywords) == 0: - raise InputError("Invalid field value %s" % v) - try: - return keywords[v.lower()] - except KeyError: - raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) - - -def createConfig(input, output): - """Parse makefile variables to create default configuration""" - - print("createConfig(%s, %s)" % (input.name, output)) - parser = configparser.ConfigParser() - parser.optionxform = str # Preserve case - data = "[DEFAULT]\r\n" + input.read().decode() - parser.read_string(data) - vars = parser['DEFAULT'] - - config = {} - config['name'] ='Generated configuration' - arch = os.environ['SMING_ARCH'] - config['arch'] = arch - if arch == 'Esp8266': - config['flash-size'] = vars['SPI_SIZE'] - config['spi-mode'] = vars['SPI_MODE'] - config['spi-speed'] = vars['SPI_SPEED'] - table = {} - config['partition-table'] = table - table['offset'] = "0x2000" - entries = {} - table['entries'] = entries - - def createEntry(address, size, type, subtype): - entry = {} - entry['address'] = '0x%06x' % address - entry['size'] = '0x%06x' % size - entry['type'] = type - entry['subtype'] = subtype - return entry - - def tryAddEntry(label, addressVar, type, subtype): - if not addressVar in vars: - return - s = vars[addressVar] - if s == '': - return None - addr = parse_int(s) - size= 0x100000 - (addr % 0x100000) - entry = createEntry(addr, size, type, subtype) - entries[label] = entry - return entry - - def addMake(entry, target): - make = {} - make['target'] = target - entry['make'] = make - return make - - def firmwareFilename(name, ext = '.bin'): - return os.environ['BUILD_BASE'] + '/' + vars[name] + ext - - if arch == 'Esp32': - entries['nvs'] = createEntry(0x9000, 0x6000, "data", "nvs") - entries['phy_init'] = createEntry(0xf000, 0x1000, "data", "phy") - entries['factory'] = createEntry(0x10000, 0x1f000, "app", "factory") - vars['RBOOT_SPIFFS_0'] = '0x100000' - else: - entries['phy_init'] = createEntry(0x3000, 0x1000, "data", "phy") - entries['sysconfig'] = createEntry(0x4000, 0x4000, "data", "nvs") - entry = tryAddEntry('rom0', 'RBOOT_ROM0_ADDR', 'app', 'factory') - if not entry is None: - entry['filename'] = firmwareFilename('RBOOT_ROM_0') - addMake(entry, 'fwimage') - entry = tryAddEntry('rom1', 'RBOOT_ROM1_ADDR', 'app', 'ota_0') - if not entry is None: - entry['filename'] = firmwareFilename('RBOOT_ROM_1') - addMake(entry, 'fwimage') - tryAddEntry('rom2', 'RBOOT_ROM2_ADDR', 'app', 'ota_1') - - entry = tryAddEntry('spiffs1', 'RBOOT_SPIFFS_0', 'data', 'spiffs') - if vars['DISABLE_SPIFFS'] == '0' and not entry is None: - entry['filename'] = firmwareFilename('SPIFF_BIN') - make = addMake(entry, 'spiffsgen') - make['content'] = vars['SPIFF_FILES'] - make['size'] = vars['SPIFF_SIZE'] - tryAddEntry('spiffs2', 'RBOOT_SPIFFS_1', 'data', 'spiffs') - - with sys.stdout if output == '-' else open(output, 'w') as f: - f.write(json.dumps(config, indent=4)) - + stdout_binary = sys.stdout.buffer # Python 3 + except AttributeError: + stdout_binary = sys.stdout + return stdout_binary + status("Writing to '%s'" % path) + output_dir = os.path.abspath(os.path.dirname(path)) + os.makedirs(output_dir, exist_ok=True) + return open(path, 'wb') def main(): - global quiet - global md5sum - global offset_part_table - global secure parser = argparse.ArgumentParser(description='Sming hardware configuration utility') - parser.add_argument('--flash-size', help='Optional flash size limit, checks partition table fits in flash', - nargs='?', choices=['1MB', '2MB', '4MB', '8MB', '16MB']) - parser.add_argument('--disable-md5sum', help='Disable md5 checksum for the partition table', default=False, action='store_true') parser.add_argument('--no-verify', help="Don't verify partition table fields", action='store_true') - parser.add_argument('--verify', '-v', help="Verify partition table fields (deprecated, this behaviour is " - "enabled by default and this flag does nothing.", action='store_true') parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true') - parser.add_argument('--offset', '-o', help='Set offset partition table', default='0x8000') parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true') - parser.add_argument('input', help='Path to JSON or binary file to parse.', type=argparse.FileType('rb')) - parser.add_argument('output', help='Path to output converted binary or CSV file. Will use stdout if omitted.', - nargs='?', default='-') + parser.add_argument('--part', help="Name of partition to operate on") + parser.add_argument('command', help='Action to perform', choices=['partgen', 'expr', 'validate']) + parser.add_argument('input', help='Name of hardware configuration or path to binary partition table') + parser.add_argument('output', help='Path to output file. Will use stdout if omitted.', nargs='?', default='-') + parser.add_argument('expr', help='Expression to evaluate', nargs='?', default=None) args = parser.parse_args() - quiet = args.quiet - md5sum = not args.disable_md5sum - secure = args.secure - offset_part_table = int(args.offset, 0) - - if args.input.name.endswith('.mk'): - createConfig(args.input, args.output) - return - - input = args.input.read() - input_is_binary = (input[0:2] == PartitionDefinition.MAGIC_BYTES) - if args.input.name.endswith('.hw'): - status("Parsing JSON input...") - config = json.loads(input.decode()) - table = PartitionTable.from_json(config['partition-table']) - elif input_is_binary: - status("Parsing binary partition input...") - table = PartitionTable.from_binary(input) - else: - raise InputError("Unknown input file format") - - if not args.no_verify: - status("Verifying table...") - table.verify() - - if args.flash_size: - size_mb = int(args.flash_size.replace("MB", "")) - size = size_mb * 1024 * 1024 # flash memory uses honest megabytes! - table_size = table.flash_size() - if size < table_size: - raise InputError("Partitions defined in '%s' occupy %.1fMB of flash (%d bytes) which does not fit in configured " - "flash size %dMB. Please change SPI_SIZE setting." % - (args.input.name, table_size / 1024.0 / 1024.0, table_size, size_mb)) - - # Make sure that the output directory is created - output_dir = os.path.abspath(os.path.dirname(args.output)) + common.quiet = args.quiet + + output = None + input_is_binary = False + if os.path.exists(args.input): + inputData = open(args.input, "rb").read() + input_is_binary = inputData[0:2] == partition.Entry.MAGIC_BYTES + if input_is_binary: + config = Config.from_binary(inputData) + if not input_is_binary: + config = Config.from_name(args.input) + partitions = config.partitions + + # Define local so it's available for eval() + if args.part is not None: + part = partitions.find_by_name(args.part) + if part is None: + return - if not os.path.exists(output_dir): + if args.command == 'validate': try: - os.makedirs(output_dir) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - - if input_is_binary: - output = table.to_csv() - with sys.stdout if args.output == '-' else open(args.output, 'w') as f: - f.write(output) + from jsonschema import Draft7Validator + from jsonschema.exceptions import ValidationError + except ImportError: + critical("hwconfig: `jsonschema` is not installed. Please run `make python-requirements`") + sys.exit(1) + inst = json.loads(config.to_json()) + schema = json.load(open(args.expr)) + v = Draft7Validator(schema) + errors = sorted(v.iter_errors(inst), key=lambda e: e.path) + if errors != []: + for e in errors: + critical("%s @ %s" % (e.message, e.path)) + sys.exit(3) + elif args.command == 'partgen': + if not args.no_verify: + status("Verifying partition table...") + config.verify(args.secure) + output = config.partitions.to_binary(config.devices) + elif args.command == 'expr': + output = str(eval(args.expr)).encode() else: - output = table.to_binary() - try: - stdout_binary = sys.stdout.buffer # Python 3 - except AttributeError: - stdout_binary = sys.stdout - with stdout_binary if args.output == '-' else open(args.output, 'wb') as f: - f.write(output) - - -class InputError(RuntimeError): - - def __init__(self, e): - super(InputError, self).__init__(e) - - -class ValidationError(InputError): + raise InputError('Unknown command: %s' % args.command) - def __init__(self, partition, message): - super(ValidationError, self).__init__( - "Partition %s invalid: %s" % (partition.name, message)) + if output is not None: + openOutput(args.output).write(output) if __name__ == '__main__': diff --git a/Sming/Components/Storage/Tools/hwconfig/partition.py b/Sming/Components/Storage/Tools/hwconfig/partition.py new file mode 100644 index 0000000000..aa0515e304 --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/partition.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +# +# From Espressif gen_esp32part.py +# +# Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import struct, hashlib, storage, binascii +from common import * + +MAX_PARTITION_LENGTH = 0xC00 # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature +MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14 # The first 2 bytes are like magic numbers for MD5 sum +PARTITION_TABLE_SIZE = 0x1000 # Size of partition table +PARTITION_ENTRY_SIZE = 32 + +MIN_PARTITION_SUBTYPE_APP_OTA = 0x10 +NUM_PARTITION_SUBTYPE_APP_OTA = 16 + +__version__ = '1.2' + +APP_TYPE = 0x00 +DATA_TYPE = 0x01 +STORAGE_TYPE = 0x02 # Reference to storage device +USER_TYPE = 0x40 # First user-defined type + +# Default is 4 +ALIGNMENT = { + "Esp32": { + APP_TYPE: 0x10000, + }, + "Esp8266": { + APP_TYPE: 0x1000, + }, + "Host": { + APP_TYPE: 0x1000, + } +} + +TYPES = { + "app": APP_TYPE, + "data": DATA_TYPE, + "storage": STORAGE_TYPE, + "user": USER_TYPE, +} + +# Keep this map in sync with esp_partition_subtype_t enum in esp_partition.h +SUBTYPES = { + APP_TYPE: { + "factory": 0x00, + "ota_0": 0x10, + "ota_1": 0x11, + "ota_2": 0x12, + "ota_3": 0x13, + "ota_4": 0x14, + "ota_5": 0x15, + "ota_6": 0x16, + "ota_7": 0x17, + "ota_8": 0x18, + "ota_9": 0x19, + "ota_10": 0x1a, + "ota_11": 0x1b, + "ota_12": 0x1c, + "ota_13": 0x1d, + "ota_14": 0x1e, + "ota_15": 0x1f, + "test": 0x20, + }, + DATA_TYPE: { + "ota": 0x00, + "phy": 0x01, + "nvs": 0x02, + "coredump": 0x03, + "nvs_keys": 0x04, + "efuse": 0x05, + "sysparam": 0x40, + "esphttpd": 0x80, + "fat": 0x81, + "spiffs": 0x82, + }, + STORAGE_TYPE: storage.TYPES +} + + +def parse_type(value): + if isinstance(value, str): + if value == "": + raise InputError("Field 'type' can't be left empty.") + return parse_int(value, TYPES) + return value + + +def parse_subtype(ptype, value): + if isinstance(value, str): + if value == "": + return 0 # default + return parse_int(value, SUBTYPES.get(ptype, {})) + return value + + +class Table(list): + + def __init__(self): + super(Table, self).__init__(self) + + def parse_dict(self, data, devices): + partnames = [] + for name, entry in data.items(): + if name in partnames: + raise InputError("Duplicate partition '%s'" % name) + if contains_whitespace(name): + raise InputError("Partition names may not contain spaces '%s'" % name) + partnames += name + part = self.find_by_name(name) + if part is None: + part = Entry(devices[0], name) + self.append(part) + part.parse_dict(entry, devices) + self.sort() + + def sort(self): + super(Table, self).sort(key=lambda p: p.device.name + p.address_str()) + + def dict(self): + res = {} + for entry in self: + res[entry.name] = entry.dict() + return res + + def to_json(self): + return to_json(self.dict()); + + def to_csv(self): + res = "Device Start End Size Type SubType Name Filename\n" + res += "---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------\n" + for p in self: + res += "%-16s %10s %10s %10s %-8s %-8s %-16s %s\n" \ + % (p.device.name, p.address_str(), p.end_str(), p.size_str(), p.type_str(), p.subtype_str(), p.name, p.filename) + return res + + def offset_str(self): + return addr_format(self.offset) + + def buildVars(self): + dict = {} + dict['PARTITION_NAMES'] = " ".join(p.name for p in self) + for p in self: + dict.update(p.buildVars()) + return dict + + def __getitem__(self, item): + """Allow partition table access by name or index + """ + if isinstance(item, str): + p = self.find_by_name(item) + if p is None: + raise ValueError("No partition entry named '%s'" % item) + return p + return super(Table, self).__getitem__(item) + + def find_by_type(self, ptype, subtype): + """Return a partition by type & subtype, returns None if not found + """ + # convert ptype & subtypes names (if supplied this way) to integer values + try: + ptype = TYPES[ptype] + except KeyError: + try: + ptype = int(ptype, 0) + except TypeError: + pass + try: + subtype = SUBTYPES[int(ptype)][subtype] + except KeyError: + try: + subtype = int(subtype, 0) + except TypeError: + pass + + for p in self: + if p.type == ptype and p.subtype == subtype: + yield p + return + + def find_by_name(self, name): + for p in self: + if p.name == name: + return p + return None + + def verify(self, arch, secure): + """Verify partition layout + """ + # verify each partition individually + for p in self: + p.verify(arch, secure) + + # check on duplicate name + names = [p.name for p in self] + duplicates = set(n for n in names if names.count(n) > 1) + + # print sorted duplicate partitions by name + if len(duplicates) != 0: + print("A list of partitions that have the same name:") + for p in sorted(self, key=lambda x:x.name): + if len(duplicates.intersection([p.name])) != 0: + print("%s" % (p.to_csv())) + raise InputError("Partition names must be unique") + + # check for overlaps + dev = '' + last = None + for p in self: + if p.device != dev: + last = None + dev = p.device + if dev == self[0].device and p.address < self.offset + PARTITION_TABLE_SIZE: + raise InputError("Partition '%s' @ %s-%s is before partition table @ %s" \ + % (p.name, p.address_str(), p.end_str(), addr_format(self.offset + PARTITION_TABLE_SIZE))) + if last is not None and p.address <= last.end(): + raise InputError("Partition '%s' @ %s-%s overlaps '%s' @ %s-%s" \ + % (p.name, p.address_str(), p.end_str(), \ + last.name, last.address_str(), last.end_str())) + last = p + + def parse_binary(self, b, devices): + """Construct partition table object from binary image + """ + dev = None + md5 = hashlib.md5() + for o in range(0, len(b), PARTITION_ENTRY_SIZE): + data = b[o:o + PARTITION_ENTRY_SIZE] + if len(data) != PARTITION_ENTRY_SIZE: + raise InputError("Partition table length must be a multiple of %d bytes" % PARTITION_ENTRY_SIZE) + if data == b'\xFF' * PARTITION_ENTRY_SIZE: + return # got end marker + + if data[:2] == MD5_PARTITION_BEGIN[:2]: # check only the magic number part + if data[16:] == md5.digest(): + md5.update(data) + continue # the next iteration will check for the end marker + else: + raise InputError("MD5 checksums don't match! (computed: 0x%s, parsed: 0x%s)" % (md5.hexdigest(), binascii.hexlify(data[16:]))) + + md5.update(data) + + if data[:2] == b"\xff\xff": # Pseudo-end marker to keep esp32 bootloader happy + continue + + e = Entry.from_binary(data) + if e.type == STORAGE_TYPE: + dev = storage.Device(e.name, e.subtype, e.size) + devices.append(dev) + else: + e.device = dev + self.append(e) + raise InputError("Partition table is missing an end-of-table marker") + + def to_binary(self, devices): + """Create binary image of partition table + """ + dev_count = 0 + dev = None + result = b"" + for e in self: + if e.device != dev: + if dev_count == 1: + result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() + # esp32 bootloader will see this as end of partition table + result += struct.pack(Entry.STRUCT_FORMAT, + b"\xff\xff", + 0xff, 0xff, + 0, 0, + b"SMING EXTENSIONS", + 0) + dev = e.device + s = Entry(dev, dev.name, 0, dev.size, STORAGE_TYPE, dev.type) + result += s.to_binary() + dev_count += 1 + result += e.to_binary() + + result += MD5_PARTITION_BEGIN + hashlib.md5(result).digest() + + if len(result) >= MAX_PARTITION_LENGTH: + raise InputError("Binary partition table length (%d) longer than max" % len(result)) + result += b"\xFF" * (MAX_PARTITION_LENGTH - len(result)) # pad the sector, for signing + return result + + +class Entry(object): + MAGIC_BYTES = b"\xAA\x50" + + # dictionary maps flag name (as used in CSV flags list, property name) + # to bit set in flags words in binary format + FLAGS = { + "encrypted": 0, + "readonly": 31, + } + + def __init__(self, device=None, name="", address=None, size=None, ptype=None, subtype=None): + self.device = device + self.name = name + self.address = address + self.size = size + self.type = parse_type(ptype) + self.subtype = parse_subtype(self.type, subtype) + self.flags = set() + self.filename = '' + self.build = None + + + def parse_dict(self, data, devices): + """Construct a partition object from JSON definition + """ + try: + v = data.get('type') + if not v is None: + self.type = parse_type(v) + v = data.get('subtype') + if not v is None: + self.subtype = parse_subtype(self.type, v) + + for k, v in data.items(): + if k == 'device': + self.device = devices.find_by_name(v) + elif k == 'address': + self.address = parse_int(v) + elif k == 'size': + self.size = parse_int(v) + elif k == 'flags': + flags = v.split() + for flag in flags: + if not flag in Entry.FLAGS: + raise InputError("Unknown flag '%s'" % flag) + self.flags |= {flag} + elif k == 'filename': + self.filename = v + elif k == 'build': + if self.build is None: + self.build = v + else: + self.build.update(v) + elif k != 'type' and k != 'subtype': + setattr(self, k, v) + + if self.type is None or self.subtype is None: + raise InputError("type/subtype missing") + if self.address is None or self.size is None: + raise InputError("address/size missing") + if self.end() >= self.device.size: + raise InputError("Partition '%s' %s-%s too big for %s size %s" \ + % (self.name, self.address_str(), self.end_str(), self.device.name, self.device.size_str())) + except InputError as e: + raise InputError("Error in partition entry '%s': %s" % (self.name, e)) + + + def dict(self): + res = {} + for k, v in self.__dict__.items(): + if k == 'device': + res[k] = v.name + elif k == 'address': + res[k] = self.address_str() + elif k == 'size': + res[k] = self.size_str() + elif k == 'type': + res[k] = stringnum(self.type_str()) + elif k == 'subtype': + res[k] = stringnum(self.subtype_str()) + elif k == 'flags': + res[k] = self.flags_str() + elif v is not None and k != 'name': + res[k] = v + return res + + def buildVars(self): + res = {} + + dict = self.dict() + dict['size_bytes'] = "0x%x" % self.size + dict.pop('build', None) + for k, v in dict.items(): + k = "PARTITION_%s_%s" % (self.name, k.upper()) + res[k] = v + + return res + + def to_json(self): + return to_json(self.dict()); + + def end(self): + return self.address + self.size - 1 + + def end_str(self): + return addr_format(self.end()) + + def address_str(self): + return addr_format(self.address) + + def size_str(self): + return size_format(self.size) + + def type_str(self): + return "" if self.type == 0xff else lookup_keyword(self.type, TYPES) + + def type_is(self, t): + return self.type_str() == t if isinstance(t, str) else self.type == t + + def subtype_str(self): + return "" if self.subtype == 0xff else lookup_keyword(self.subtype, SUBTYPES.get(self.type, {})) + + def subtype_is(self, subtype): + return self.subtype_str() == subtype if isinstance(subtype, str) else self.subtype == subtype + + def flags_str(self): + return " ".join(self.flags) + + def __eq__(self, other): + if isinstance(other, str): + return self.name == other + else: + return self.name == other.name and self.type == other.type \ + and self.subtype == other.subtype and self.address == other.address \ + and self.size == other.size + + def __repr__(self): + + def maybe_hex(x): + return "0x%x" % x if x is not None else "None" + + return "Entry('%s', 0x%x, 0x%x, %s, %s)" % (self.name, self.type, self.subtype or 0, + maybe_hex(self.address), maybe_hex(self.size)) + + def __str__(self): + return "Part '%s' %s/%s @ 0x%x size 0x%x" % (self.name, self.type_str(), self.subtype_str(), self.address or -1, self.size or -1) + + def alignment(self, arch): + return ALIGNMENT[arch].get(self.type, 4) + + def verify(self, arch, secure): + if self.type is None: + raise ValidationError(self, "Type field is not set") + if self.subtype is None: + raise ValidationError(self, "Subtype field is not set") + if self.address is None: + raise ValidationError(self, "Address field is not set") + align = self.alignment(arch) + if self.address % align: + raise ValidationError(self, "Offset 0x%x is not aligned to 0x%x" % (self.address, align)) + if self.size % align and secure: + raise ValidationError(self, "Size 0x%x is not aligned to 0x%x" % (self.size, align)) + if self.size is None: + raise ValidationError(self, "Size field is not set") + + if self.name in TYPES and TYPES.get(self.name, "") != self.type: + critical("WARNING: Partition has name '%s' which is a partition type, but does not match this partition's " + "type (0x%x). Mistake in partition table?" % (self.name, self.type)) + all_subtype_names = [] + for names in (t.keys() for t in SUBTYPES.values()): + all_subtype_names += names + if self.name in all_subtype_names and SUBTYPES.get(self.type, {}).get(self.name, "") != self.subtype: + critical("WARNING: Partition has name '%s' which is a partition subtype, but this partition has " + "non-matching type 0x%x and subtype 0x%x. Mistake in partition table?" % (self.name, self.type, self.subtype)) + + STRUCT_FORMAT = b"<2sBBLL16sL" + + @classmethod + def from_binary(cls, b): + if len(b) != PARTITION_ENTRY_SIZE: + raise InputError("Partition entry size incorrect. Expected %d bytes, got %d." % (PARTITION_ENTRY_SIZE, len(b))) + res = cls() + (magic, res.type, res.subtype, res.address, + res.size, res.name, flags) = struct.unpack(cls.STRUCT_FORMAT, b) + if b"\x00" in res.name: # strip null byte padding from name string + res.name = res.name[:res.name.index(b"\x00")] + res.name = res.name.decode() + if magic != cls.MAGIC_BYTES: + raise InputError("Invalid magic bytes (%r) for partition definition" % magic) + for flag, bit in cls.FLAGS.items(): + if flags & (1 << bit): + setattr(res, flag, True) + flags &= ~(1 << bit) + if flags != 0: + critical("WARNING: Partition definition had unknown flag(s) 0x%08x. Newer binary format?" % flags) + return res + + def to_binary(self): + flags = sum((1 << self.FLAGS[flag]) for flag in self.flags) + return struct.pack(self.STRUCT_FORMAT, + self.MAGIC_BYTES, + self.type, self.subtype, + self.address, self.size, + self.name.encode(), + flags) + + +class Map(Table): + """Contiguous map of flash memory + """ + def __init__(self, table, devices): + device = devices[0] + + def add(name, address, size): + entry = Entry(device, name, address, size, 0xff, 0xff) + self.append(entry) + return entry + + def add_unused(address, last_end): + return add('(unused)', last_end + 1, address - last_end - 1) + + if table.offset == 0: + last = None + else: + add("Boot Sector", 0, table.offset) + last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE) + + for p in table: + if last is not None: + if p.device != last.device: + add_unused(last.device.size, last.end()) + device = p.device + elif p.address > last.end() + 1: + add_unused(p.address, last.end()) + self.append(p) + last = p + + if last.end() + 1 < device.size: + add_unused(device.size, last.end()) diff --git a/Sming/Components/Storage/Tools/hwconfig/storage.py b/Sming/Components/Storage/Tools/hwconfig/storage.py new file mode 100644 index 0000000000..719f8393dc --- /dev/null +++ b/Sming/Components/Storage/Tools/hwconfig/storage.py @@ -0,0 +1,116 @@ +# +# Storage devices +# + +from common import * + +TYPES = { + "unknown": 0x00, + "flash": 0x01, + "spiram": 0x02, + "sdcard": 0x03, + "disk": 0x04, + "file": 0x05, +} + + +def parse_type(value): + if value == "": + raise InputError("Field 'type' can't be left empty.") + return parse_int(value, TYPES) + + +class List(list): + def parse_dict(self, data): + for name, v in data.items(): + if contains_whitespace(name): + raise InputError("Device names may not contain spaces '%s'" % name) + dev = self.find_by_name(name) + if dev is None: + dev = Device(name) + self.append(dev) + dev.parse_dict(v) + + def dict(self): + res = {} + for dev in self: + res[dev.name] = dev.dict() + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + dict = {} + dict['STORAGE_DEVICE_NAMES'] = " ".join(p.name for p in self) + for p in self: + dict.update(p.buildVars()) + return dict + + def __getitem__(self, item): + if isinstance(item, str): + d = self.find_by_name(item) + if d is None: + raise ValueError("No device named '%s'" % item) + return d + return super(List, self).__getitem__(item) + + def find_by_name(self, name): + for d in self: + if d.name == name: + return d + return None + + +class Device(object): + def __init__(self, name, stype = 0, size = 0): + self.name = name + self.type = parse_type(stype) + self.size = parse_int(size) + + def parse_dict(self, data): + for k, v in data.items(): + if k == 'type': + self.type = parse_int(v, TYPES) + elif k == 'size': + self.size = parse_int(v) + elif k == 'mode': + self.mode = v + elif k == 'speed': + self.speed = v + else: + raise InputError("Unknown storage field '%s'" % k) + + def dict(self): + res = {} + + # Some fields are optional + def tryAdd(k): + if hasattr(self, k): + res[k] = getattr(self, k) + + res['type'] = self.type_str() + res['size'] = self.size_str() + tryAdd('mode') + tryAdd('speed') + return res + + def to_json(self): + return to_json(self.dict()) + + def buildVars(self): + res = {} + + dict = self.dict() + dict['size_bytes'] = "0x%x" % self.size + for k, v in dict.items(): + k = "STORAGE_DEVICE_%s_%s" % (self.name, k.upper()) + res[k] = v + + return res + + def type_str(self): + return "" if self.type == 0 else lookup_keyword(self.type, TYPES) + + def size_str(self): + return size_format(self.size) diff --git a/Sming/Components/esptool/blank.bin b/Sming/Components/Storage/blank.bin similarity index 100% rename from Sming/Components/esptool/blank.bin rename to Sming/Components/Storage/blank.bin diff --git a/Sming/Components/Storage/requirements.txt b/Sming/Components/Storage/requirements.txt new file mode 100644 index 0000000000..d89304b1a8 --- /dev/null +++ b/Sming/Components/Storage/requirements.txt @@ -0,0 +1 @@ +jsonschema diff --git a/Sming/Components/Storage/schema.json b/Sming/Components/Storage/schema.json new file mode 100644 index 0000000000..8f059cb95e --- /dev/null +++ b/Sming/Components/Storage/schema.json @@ -0,0 +1,172 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/Config", + "definitions": { + "Config": { + "title": "Hardware Configuration", + "description": "Defines memory devices and partitions for a specific hardware implementation", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "base_config": { + "type": "string", + "title": "Base configuration", + "description": "Inherit a previously-defined configuration" + }, + "comment": { + "type": "string" + }, + "arch": { + "type": "string", + "title": "Target architecture", + "description": "Defined *only* in the base 'standard' spec" + }, + "partition_table_offset": { + "type": "string", + "description": "Location of partition table in spiFlash" + }, + "devices": { + "$ref": "#/definitions/Devices" + }, + "partitions": { + "$ref": "#/definitions/Partitions" + } + }, + "required": [ + "name", + "arch", + "partition_table_offset", + "devices", + "partitions" + ] + }, + "Devices": { + "title": "Devices", + "type": "object", + "additionalProperties": false, + "properties": { + "spiFlash": { + "$ref": "#/definitions/Device", + "description": "Main flash memory device" + } + }, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "type": "object", + "$ref": "#/definitions/Device" + } + }, + "required": [ + "spiFlash" + ] + }, + "Device": { + "title": "Storage device definition", + "type": "object", + "additionalProperties": false, + "properties": { + "size": { + "type": "string" + }, + "type": { + "type": "string" + }, + "mode": { + "type": "string", + "enum": [ + "qio", + "qout", + "dio", + "dout" + ] + }, + "speed": { + "type": "integer" + } + }, + "required": [ + "size", + "type" + ] + }, + "Partitions": { + "title": "Partitions", + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_]*$": { + "type": "object", + "$ref": "#/definitions/Partition" + } + } + }, + "Partition": { + "title": "Partition definition", + "type": "object", + "additionalProperties": false, + "properties": { + "device": { + "type": "string", + "description": "ID of device this partition relates to" + }, + "address": { + "type": [ + "string", + "integer" + ] + }, + "size": { + "type": [ + "string", + "integer" + ] + }, + "type": { + "type": [ + "string", + "integer" + ] + }, + "subtype": { + "type": [ + "string", + "integer" + ] + }, + "flags": { + "type": "string" + }, + "filename": { + "type": "string", + "description": "Location of file to write to this partition" + }, + "build": { + "type": "object", + "$ref": "#/definitions/Build", + "description": "If present, used to build 'filename'" + } + }, + "required": [ + "address", + "size", + "type", + "subtype" + ] + }, + "Build": { + "title": "Build specification", + "description": "Additional properties as required by build target", + "type": "object", + "additionalProperties": true, + "properties": { + "target": { + "type": "string", + "description": "Makefile target for this build" + } + } + } + } +} \ No newline at end of file diff --git a/Sming/Components/spiffs/blankfs.bin b/Sming/Components/spiffs/blankfs.bin deleted file mode 100644 index 171fa2bbce..0000000000 --- a/Sming/Components/spiffs/blankfs.bin +++ /dev/null @@ -1 +0,0 @@ -���������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������������� \ No newline at end of file diff --git a/Sming/spiffs.hw b/Sming/spiffs.hw new file mode 100644 index 0000000000..65df953835 --- /dev/null +++ b/Sming/spiffs.hw @@ -0,0 +1,18 @@ +{ + "name": "Single SPIFFS partition", + "base_config": "standard-4m", + "partitions": { + "spiffs0": { + "address": "0x200000", + "size": "512K", + "type": "data", + "subtype": "spiffs", + "flags": "", + "filename": "$(SPIFF_BIN_OUT)", + "build": { + "target": "spiffsgen", + "files": "$(SPIFF_FILES)" + } + } + } +} \ No newline at end of file diff --git a/Sming/standard-4m.hw b/Sming/standard-4m.hw new file mode 100644 index 0000000000..c259654b19 --- /dev/null +++ b/Sming/standard-4m.hw @@ -0,0 +1,9 @@ +{ + "name": "Standard config with 4M flash", + "base_config": "standard", + "devices": { + "spiFlash": { + "size": "4M" + } + } +} From 245abf3282f8b48d8d0a170ee78ef897fe814af0 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 6 Dec 2020 20:42:35 +0000 Subject: [PATCH 08/36] Fix parse_int() function --- .../Storage/Tools/hwconfig/common.py | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Sming/Components/Storage/Tools/hwconfig/common.py b/Sming/Components/Storage/Tools/hwconfig/common.py index 8376eae740..8c28db32bb 100644 --- a/Sming/Components/Storage/Tools/hwconfig/common.py +++ b/Sming/Components/Storage/Tools/hwconfig/common.py @@ -29,18 +29,20 @@ def parse_int(v, keywords=None): """Generic parser for integer fields - int(x,0) with provision for k/m/K/M suffixes and 'keyword' value lookup. """ - try: - for letter, multiplier in [("k", 1024), ("m", 1024 * 1024)]: - if v.lower().endswith(letter): - return parse_int(v[:-1], keywords) * multiplier - return int(v, 0) - except ValueError: - if keywords is None: - raise InputError("Invalid field value %s" % v) + if not isinstance(v, str): + return v + if keywords is None or len(keywords) == 0: try: - return keywords[v.lower()] - except KeyError: - raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) + for letter, multiplier in [("k", 1024), ("m", 1024 * 1024), ("g", 1024 * 1024 * 1024)]: + if v.lower().endswith(letter): + return parse_int(v[:-1], keywords) * multiplier + return int(v, 0) + except ValueError: + raise InputError("Invalid field value %s" % v) + try: + return keywords[v.lower()] + except KeyError: + raise InputError("Value '%s' is not valid. Known keywords: %s" % (v, ", ".join(keywords))) def stringnum(s): From ebe1821e61935775d7ba3777aaa9e3fcb2e204f9 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Thu, 4 Feb 2021 20:18:48 +0000 Subject: [PATCH 09/36] Add optional tag override for PrintVariables, PrintVariableSorted and PrintVariableRefs --- Sming/build.mk | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Sming/build.mk b/Sming/build.mk index 98f2889422..d252d0f730 100644 --- a/Sming/build.mk +++ b/Sming/build.mk @@ -280,20 +280,22 @@ endef # Display variable and list values, e.g. $(call PrintVariable,LIBS) # $1 -> Name of variable containing values +# $2 -> (optional) tag to use instead of variable name define PrintVariable - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$($1),$(info - $(item))) endef define PrintVariableSorted - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$(sort $($1)),$(info - $(value item))) endef # Display list of variable references with their values e.g. $(call PrintVariableRefs,DEBUG_VARS) # $1 -> Name of variable containing list of variable names +# $2 -> (optional) tag to use instead of variable name define PrintVariableRefs - $(info $1) + $(info $(if $2,$2,$1):) $(foreach item,$(sort $($1)),$(info - $(item) = $(value $(item))) ) endef From e3f594322618a4a704e8a207a50ca0dea1a95c42 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Tue, 8 Dec 2020 08:22:39 +0000 Subject: [PATCH 10/36] Implement Partition API Add esp_flash_partitions.h from ESP-IDF (abridged) Use 'name' consistently instead of 'label' Add Storage::Device and custom device support Add `readonly` flag and device blockSize() --- Sming/Components/Storage/README.rst | 189 +++++++++ Sming/Components/Storage/component.mk | 109 +++++ Sming/Components/Storage/src/CustomDevice.cpp | 42 ++ Sming/Components/Storage/src/Device.cpp | 112 ++++++ Sming/Components/Storage/src/Iterator.cpp | 68 ++++ Sming/Components/Storage/src/ObjectList.cpp | 65 +++ Sming/Components/Storage/src/Partition.cpp | 251 ++++++++++++ .../Storage/src/PartitionStream.cpp | 61 +++ .../Components/Storage/src/PartitionTable.cpp | 37 ++ Sming/Components/Storage/src/ProgMem.cpp | 35 ++ Sming/Components/Storage/src/SpiFlash.cpp | 66 +++ Sming/Components/Storage/src/Storage.cpp | 75 ++++ Sming/Components/Storage/src/SysMem.cpp | 25 ++ .../Components/Storage/src/include/Storage.h | 62 +++ .../src/include/Storage/CustomDevice.h | 35 ++ .../Storage/src/include/Storage/Device.h | 140 +++++++ .../Storage/src/include/Storage/Iterator.h | 78 ++++ .../Storage/src/include/Storage/Object.h | 144 +++++++ .../Storage/src/include/Storage/ObjectList.h | 137 +++++++ .../Storage/src/include/Storage/Partition.h | 376 ++++++++++++++++++ .../src/include/Storage/PartitionStream.h | 61 +++ .../src/include/Storage/PartitionTable.h | 92 +++++ .../Storage/src/include/Storage/ProgMem.h | 89 +++++ .../Storage/src/include/Storage/SpiFlash.h | 38 ++ .../src/include/Storage/StreamDevice.h | 67 ++++ .../Storage/src/include/Storage/SysMem.h | 88 ++++ .../src/include/Storage/partition_info.h | 32 ++ Sming/Core/SmingCore.h | 2 + Sming/component.mk | 1 + Sming/project.mk | 1 + 30 files changed, 2578 insertions(+) create mode 100644 Sming/Components/Storage/README.rst create mode 100644 Sming/Components/Storage/component.mk create mode 100644 Sming/Components/Storage/src/CustomDevice.cpp create mode 100644 Sming/Components/Storage/src/Device.cpp create mode 100644 Sming/Components/Storage/src/Iterator.cpp create mode 100644 Sming/Components/Storage/src/ObjectList.cpp create mode 100644 Sming/Components/Storage/src/Partition.cpp create mode 100644 Sming/Components/Storage/src/PartitionStream.cpp create mode 100644 Sming/Components/Storage/src/PartitionTable.cpp create mode 100644 Sming/Components/Storage/src/ProgMem.cpp create mode 100644 Sming/Components/Storage/src/SpiFlash.cpp create mode 100644 Sming/Components/Storage/src/Storage.cpp create mode 100644 Sming/Components/Storage/src/SysMem.cpp create mode 100644 Sming/Components/Storage/src/include/Storage.h create mode 100644 Sming/Components/Storage/src/include/Storage/CustomDevice.h create mode 100644 Sming/Components/Storage/src/include/Storage/Device.h create mode 100644 Sming/Components/Storage/src/include/Storage/Iterator.h create mode 100644 Sming/Components/Storage/src/include/Storage/Object.h create mode 100644 Sming/Components/Storage/src/include/Storage/ObjectList.h create mode 100644 Sming/Components/Storage/src/include/Storage/Partition.h create mode 100644 Sming/Components/Storage/src/include/Storage/PartitionStream.h create mode 100644 Sming/Components/Storage/src/include/Storage/PartitionTable.h create mode 100644 Sming/Components/Storage/src/include/Storage/ProgMem.h create mode 100644 Sming/Components/Storage/src/include/Storage/SpiFlash.h create mode 100644 Sming/Components/Storage/src/include/Storage/StreamDevice.h create mode 100644 Sming/Components/Storage/src/include/Storage/SysMem.h create mode 100644 Sming/Components/Storage/src/include/Storage/partition_info.h diff --git a/Sming/Components/Storage/README.rst b/Sming/Components/Storage/README.rst new file mode 100644 index 0000000000..b80c0f611c --- /dev/null +++ b/Sming/Components/Storage/README.rst @@ -0,0 +1,189 @@ +Storage Manager +=============== + +This Component provides support for using storage devices in a structured way by partitioning +them into areas for specific uses. + +Partitions may can contain information such as: + +- Application (firmware) images +- Filesystem(s) +- Configuration/calibration/parameter data +- Custom flash storage areas + +A single partition table is located on the main flash device, :cpp:var:`Storage::spiFlash`, +and defines all partitions with a unique name and associated :cpp:type:`Storage::Partition::Type` / :cpp:type:`Storage::Partition::SubType`. + + + +.. _hardware_config: + +Hardware configuration +---------------------- + +Each project has an associated ``Hardware configuration``, specified by the :envvar:`HWCONFIG` setting. + +This information is defined in one or more JSON files, identified with a ``.hw`` extension. +You can find a schema for this file in :doc:`schema.json`. + +The build system locates the file by searching, in order: + +- ``{PROJECT_DIR}`` the root project directory +- ``{SMING_HOME}/Arch/{SMING_ARCH}`` +- ``{SMING_HOME}`` + +Files (other than standard) will have a ``base_config`` setting. The specified profile is pulled +in following the same search pattern as above. + +This provides a mechanism for inheritance to allow adjustment of existing profiles +without having to copy it entirely. + +For example, to adjust the address/size of the main application partition +(for the esp8266, this is ``rom0``), you can do this: + +.. code-block:: json + + { + "name": "My config", + "base_config": "standard-4m", + "partitions": { + "rom0": { + "address": "0x10000", + "size": "0x80000" + } + } + } + + +You can examine your project's active configuration like this:: + + make hwconfig + +This also shows a list of the discovered configurations available. + +You can also display the memory map:: + + make map + +Which, for Host arch, produces: + +.. code-block:: text + + HostTests: Invoking 'map' for Host (debug) architecture + Device Start End Size Type SubType Name Filename + ---------------- -------- -------- -------- -------- -------- ---------------- ------------ + spiFlash 0x000000 0x001fff 8K Boot Sector + spiFlash 0x002000 0x002fff 4K Partition Table + spiFlash 0x003000 0x007fff 20K (unused) + spiFlash 0x008000 0x0fffff 992K app factory rom0 $(BLANK_BIN) + spiFlash 0x100000 0x1fffff 1M (unused) + spiFlash 0x200000 0x27ffff 512K data spiffs spiffs0 $(FW_BASE)/spiffs0.bin + spiFlash 0x280000 0x3fffff 1536K (unused) + test-device 0x000000 0x3fffff 4M data spiffs external1 + test-device 0x400000 0x5fffff 2M (unused) + test-device 0x600000 0x63bfff 240K data 37 external2 + test-device 0x63c000 0xffffff 10000K (unused) + +For comparison, here's the output for Esp32: + +.. code-block:: text + + HostTests: Invoking 'map' for Esp32 (debug) architecture + Device Start End Size Type SubType Name Filename + ---------------- -------- -------- -------- -------- -------- ---------------- ------------ + spiFlash 0x000000 0x007fff 32K Boot Sector + spiFlash 0x008000 0x008fff 4K Partition Table + spiFlash 0x009000 0x00efff 24K data nvs nvs + spiFlash 0x00f000 0x00ffff 4K data phy phy_init + spiFlash 0x010000 0x1fffff 1984K app factory factory $(TARGET_BIN) + spiFlash 0x200000 0x27ffff 512K data spiffs spiffs0 $(FW_BASE)/spiffs0.bin + spiFlash 0x280000 0x3fffff 1536K (unused) + test-device 0x000000 0x3fffff 4M data spiffs external1 + test-device 0x400000 0x5fffff 2M (unused) + test-device 0x600000 0x63bfff 240K data 37 external2 + test-device 0x63c000 0xffffff 10000K (unused) + + +Configuration +------------- + +.. envvar:: HWCONFIG + + default: standard + + Set this to the hardware configuration file to use for your project (without .hw extension) + + Default configurations: + + standard + Base profile which should work on all device variants. + Located in the ``Sming/Arch/{SMING_ARCH}`` directory. + standard-4m + Supports only devices with 4Mbyte flash + spiffs + Based on standard-4m and adds a single SPIFFS partition + + Other configurations may be available, depending on architecture. + + You can see these by running ``make hwconfig``. + + +Binary partition table +---------------------- + +Sming uses the same binary partition table structure as ESP-IDF, located immediately after the boot sector. +However, it is organised slighly differently to allow partitions to be registered for multiple storage devices. + +Entries are fixed 32-byte structures, :cpp:class:`Storage::esp_partition_info_t`, organised as follows: + +- The first entry is always a ``storage`` type defining the main :cpp:var:`spiFlash` device. +- This is followed by regular partition entries sorted in ascending address order. + There may be gaps between the partitions. +- The partition table md5sum entry is inserted as normal +- If any external devices are defined: + - A SMING_EXTENSION entry, which the esp32 bootloader interprets as the end of the partition table. + - The next entry is a ``storage`` type for the 'external` device. + - This is followed by regular partition entries as before. + - A second md5sum entry is inserted for the entire partition table thus far +- The end of the partition table is identified by an empty sector (i.e. all bytes 0xFF). + +On initialisation, partition entries with the correct MAGIC entry are loaded and all others are ignored. + +A storage partition entry `address` field is always initialised to 0. +When :cpp:func:`PartitionTable::registerStorageDevice()` is called, this field is updated with +a pointer to the :cpp:class:`Storage::Device` implementation. + +Located :cpp:class:`Partition` objects contain a reference to both the requested partition +entry and the storage partition entry. + + +Partition API +------------- + +This is a C++ interface. Some examples:: + + auto part = partitionTable.find("spiffs0"); // Find by name + + // Enumerate all partitions + for(auto part: partitionTable) { + debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + } + + // Enumerate all SPIFFS partitions + for(auto it = partitionTable.find(Partition::SubType::Data::spiffs; it; it++) { + auto part = *it; + debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + } + + +A :cpp:class:`Storage::Partition` object is just a wrapper and can be freely copied around. +It defines methods which should be used to read/write/erase the partition contents. + +Each partition has an associated :cpp:class:`Storage::Device`. +This is usually :cpp:var:`Storage::spiFlash` for the main flash device. + +Other devices must be registed via :cpp:func:`Storage::PartitionTable::registerStorageDevice`. + + +.. doxygennamespace:: Storage + :members: diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk new file mode 100644 index 0000000000..ce22e5e331 --- /dev/null +++ b/Sming/Components/Storage/component.mk @@ -0,0 +1,109 @@ +COMPONENT_INCDIRS := src/include +COMPONENT_SRCDIRS := src +COMPONENT_DOXYGEN_INPUT := src/include + +CONFIG_VARS += HWCONFIG +ifndef HWCONFIG +override HWCONFIG := standard +$(info Using configuration '$(HWCONFIG)') +endif + +HWCONFIG_DIRS := $(PROJECT_DIR) $(COMPONENT_SEARCH_DIRS) $(ARCH_BASE) $(SMING_HOME) +HWCONFIG_PATH := $(firstword $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/$(HWCONFIG).hw))) +ALL_HWCONFIG := $(sort $(notdir $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/*.hw)))) +ALL_HWCONFIG := $(ALL_HWCONFIG:.hw=) + +ifeq (,$(wildcard $(HWCONFIG_PATH))) +$(info $(HWCONFIG_DIRS)) +$(eval $(call PrintVariable,ALL_HWCONFIG)) +$(error Hardware configuration '$(HWCONFIG)' not found) +endif + +PARTITION_PATH := $(COMPONENT_PATH) +PARTITION_TOOLS := $(PARTITION_PATH)/Tools +HWCONFIG_TOOL := \ + HWCONFIG_DIRS="$(HWCONFIG_DIRS)" \ + BUILD_BASE=$(BUILD_BASE) \ + $(PYTHON) $(PARTITION_TOOLS)/hwconfig/hwconfig.py + +ifeq (,$(MAKE_DOCS)) + +HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part $(PART)) expr $(HWCONFIG) - + +define HwExpr +$(shell $(HWEXPR) "$1") +endef + +# Obtain partition information +# $1 -> Partition name +# $2 -> Expression +define PartInfo +$(shell $(HWCONFIG_TOOL) --part $1 expr $(HWCONFIG) - $2) +endef + + +# Import PARTITION_TABLE_OFFSET from hardware configuration +DEBUG_VARS += PARTITION_TABLE_OFFSET +SMING_ARCH_HW := $(call HwExpr,config.arch) +ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) +$(error Hardware configuration '$(HWCONFIG)' is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) +endif +PARTITION_TABLE_OFFSET := $(call HwExpr,(config.partitions.offset_str())) +COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) + + +##@Configuration + +.PHONY: map +map: $(HWCONFIG_PATH) ##Show memory map + $(Q) $(HWEXPR) "config.map().to_csv()" + +.PHONY: hwexpr +hwexpr: $(HWCONFIG_PATH) ##Evaluate expression against hardware configuration (use EXPR= and optionally PART=) + $(Q) $(HWEXPR) "$(EXPR)" + +.PHONY: hwconfig +hwconfig: $(HWCONFIG_PATH) ##Show current hardware configuration + @echo + @echo Hardware Configuration + @echo Available: $(ALL_HWCONFIG) + @echo Selected: $(HWCONFIG) + $(Q) $(HWEXPR) "config.to_json()" + @echo + +.PHONY: hwconfig-create +hwconfig-create: ##Create hardware configuration from current build variables + $(Q) $(HWCONFIG_TOOL) create-config $(CONFIG_CACHE_FILE) - + + +##@Building + +# The partition table +PARTITIONS_BIN := $(FW_BASE)/partitions.bin +CUSTOM_TARGETS += $(PARTITIONS_BIN) + +$(PARTITIONS_BIN): $(HWCONFIG_PATH) + $(Q) $(HWCONFIG_TOOL) partgen $(HWCONFIG) $@ + + +##@Flashing + +# $1 -> Filter expression +define PartChunks +$(call HwExpr,(' '.join(['%s=%s' % (part.address_str(), part.filename) for part in filter(lambda part: $1 and part.filename != '', config.partitions)]))) +endef + +# One flash sector of 0xFF +BLANK_BIN := $(PARTITION_PATH)/blank.bin + +DEBUG_VARS += FLASH_APP_CHUNKS FLASH_PARTITION_CHUNKS +$(eval FLASH_APP_CHUNKS = $(call PartChunks,(part.type_is('app')))) +$(eval FLASH_PARTITION_CHUNKS = $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) $(call PartChunks,True)) + +ifdef PART +DEBUG_VARS += FLASH_PART_CHUNKS +$(eval FLASH_PART_CHUNKS = $(call PartChunks,(part=='$(PART)'))) +endif + + +endif # MAKE_DOCS diff --git a/Sming/Components/Storage/src/CustomDevice.cpp b/Sming/Components/Storage/src/CustomDevice.cpp new file mode 100644 index 0000000000..a960528961 --- /dev/null +++ b/Sming/Components/Storage/src/CustomDevice.cpp @@ -0,0 +1,42 @@ +/* + * CustomDevice.cpp + */ + +#include "include/Storage/CustomDevice.h" +#include + +namespace Storage +{ +namespace +{ +static constexpr size_t maxPartitions{16}; ///< Hard limit on partition table size + +class Partitions : public PartitionTable +{ +public: + Partition add(const Partition::Info& info) + { + if(!mEntries) { + mEntries.reset(new Partition::Info[maxPartitions]); + } else + assert(mCount < maxPartitions); + + auto i = mCount++; + mEntries.get()[i] = info; + return operator[](i); + } +}; + +} // namespace + +Partition CustomDevice::createPartition(const Partition::Info& info) +{ + if(mPartitions.count() >= maxPartitions) { + debug_e("Partition table is full for '%s'", getName().c_str()); + return Partition{}; + } + + return reinterpret_cast(mPartitions).add(info); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Device.cpp b/Sming/Components/Storage/src/Device.cpp new file mode 100644 index 0000000000..6f2c6ec428 --- /dev/null +++ b/Sming/Components/Storage/src/Device.cpp @@ -0,0 +1,112 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Device.h - external storage device API + * + ****/ + +#include "include/Storage.h" +#include "include/Storage/Device.h" +#include "include/Storage/partition_info.h" +#include +#include + +namespace +{ +#define XX(type, value, desc) DEFINE_FSTR_LOCAL(typestr_##type, #type) +STORAGE_TYPE_MAP(XX) +#undef XX + +#define XX(type, value, desc) &typestr_##type, +DEFINE_FSTR_VECTOR_LOCAL(typeStrings, FlashString, STORAGE_TYPE_MAP(XX)) +#undef XX + +#define XX(type, value, desc) DEFINE_FSTR_LOCAL(long_typestr_##type, desc) +STORAGE_TYPE_MAP(XX) +#undef XX + +#define XX(type, value, desc) &long_typestr_##type, +DEFINE_FSTR_VECTOR_LOCAL(longTypeStrings, FlashString, STORAGE_TYPE_MAP(XX)) +#undef XX + +} // namespace + +String toString(Storage::Device::Type type) +{ + return typeStrings[unsigned(type)]; +} + +String toLongString(Storage::Device::Type type) +{ + return longTypeStrings[unsigned(type)]; +} + +namespace Storage +{ +Device::~Device() +{ + unRegisterDevice(this); +} + +bool Device::loadPartitions(Device& source, uint32_t tableOffset) +{ + constexpr size_t maxEntries = ESP_PARTITION_TABLE_MAX_LEN / sizeof(esp_partition_info_t); + esp_partition_info_t buffer[maxEntries]; + if(!source.read(tableOffset, buffer, sizeof(buffer))) { + debug_e("[Partition] Failed to read partition table at offset 0x%08x", tableOffset); + return false; + } + + if(buffer[0].type != Partition::Type::storage) { + debug_e("[Partition] Bad partition table for device '%s' @ 0x%08x", getName().c_str(), tableOffset); + return false; + } + + String devname = getName(); + for(unsigned i = 0; i < maxEntries; ++i) { + auto entry = &buffer[i]; + if(entry->magic != ESP_PARTITION_MAGIC) { + continue; + } + if(entry->type != Partition::Type::storage) { + continue; + } + + auto len = devname.length(); + if(len > Partition::nameSize) { + continue; + } + if(strncmp(entry->name, devname.c_str(), len) != 0) { + continue; + } + + if(entry->subtype != uint8_t(getType())) { + debug_w("[Device] '%s' type mismatch, '%s' in partition table but device reports '%s'", getName().c_str(), + toString(Device::Type(entry->subtype)).c_str(), toString(getType()).c_str()); + } + if(entry->size != getSize()) { + debug_w("[Device] '%s' size mismatch, 0x%08x in partition table but device reports 0x%08x", + getName().c_str(), entry->size, getSize()); + } + + // Skip the storage entry, not required + ++entry; + ++i; + unsigned count{0}; + while(i < maxEntries && buffer[i].magic == ESP_PARTITION_MAGIC) { + ++i; + ++count; + } + + mPartitions.load(entry, count); + return true; + } + + // No partitions found + return false; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Iterator.cpp b/Sming/Components/Storage/src/Iterator.cpp new file mode 100644 index 0000000000..a082b4778d --- /dev/null +++ b/Sming/Components/Storage/src/Iterator.cpp @@ -0,0 +1,68 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Iterator.cpp + * + ****/ + +#include "include/Storage/Iterator.h" +#include "include/Storage/SpiFlash.h" +#include + +namespace Storage +{ +Iterator::Iterator(Device& device, uint8_t partitionIndex) + : mSearch{&device, Partition::Type::any, Partition::SubType::any}, mDevice(&device), mPos(partitionIndex) + +{ + if(partitionIndex >= device.partitions().count()) { + mDevice = nullptr; + mPos = afterEnd; + } +} + +Iterator::Iterator(Partition::Type type, uint8_t subtype) : mSearch{nullptr, type, subtype} +{ + mDevice = spiFlash; + next(); +} + +bool Iterator::next() +{ + while(mDevice != nullptr) { + while(uint8_t(++mPos) < mDevice->partitions().count()) { + auto entry = mDevice->partitions()[mPos]; + + if(mSearch.type != Partition::Type::any && mSearch.type != entry.type()) { + continue; + } + + if(mSearch.subType != Partition::SubType::any && mSearch.subType != entry.subType()) { + continue; + } + + return true; + } + + mPos = afterEnd; + if(mSearch.device != nullptr) { + mDevice = nullptr; + break; + } + + mDevice = mDevice->getNext(); + mPos = beforeStart; + } + + return false; +} + +Partition Iterator::operator*() const +{ + return mDevice ? mDevice->partitions()[mPos] : Partition{}; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/ObjectList.cpp b/Sming/Components/Storage/src/ObjectList.cpp new file mode 100644 index 0000000000..dae8a07716 --- /dev/null +++ b/Sming/Components/Storage/src/ObjectList.cpp @@ -0,0 +1,65 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * ObjectList.cpp + * + ****/ + +#include "include/Storage/ObjectList.h" + +namespace Storage +{ +bool ObjectList::add(Object* object) +{ + if(object == nullptr) { + return false; + } + + Object* prev = nullptr; + auto it = mHead; + while(it != nullptr) { + if(it == object) { + // Already in list + return true; + } + prev = it; + it = it->mNext; + } + + if(prev == nullptr) { + mHead = object; + } else { + prev->mNext = object; + } + object->mNext = it; + return true; +} + +bool ObjectList::remove(Object* object) +{ + if(object == nullptr) { + return false; + } + + if(mHead == object) { + mHead = object->mNext; + return true; + } + + auto it = mHead; + while(it->mNext != nullptr) { + if(it->mNext == object) { + it->mNext = object->mNext; + object->mNext = nullptr; + return true; + } + it = it->mNext; + } + + return false; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Partition.cpp b/Sming/Components/Storage/src/Partition.cpp new file mode 100644 index 0000000000..d66c128fbb --- /dev/null +++ b/Sming/Components/Storage/src/Partition.cpp @@ -0,0 +1,251 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Partition.cpp - Partition support for all architectures + * + ****/ + +#include "include/Storage/Partition.h" +#include "include/Storage/Device.h" +#include +#include + +using namespace Storage; + +namespace +{ +/* APP type strings */ + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(app_subTypeStr_##subtype, #subtype) +PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::App::subtype, &app_subTypeStr_##subtype}, +DEFINE_FSTR_MAP_LOCAL(appSubTypeStrings, Partition::SubType::App, FlashString, PARTITION_APP_SUBTYPE_MAP(XX)) +#undef XX + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(app_subTypeStr_long_##subtype, desc) +PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::App::subtype, &app_subTypeStr_long_##subtype}, +DEFINE_FSTR_MAP_LOCAL(longAppSubTypeStrings, Partition::SubType::App, FlashString, PARTITION_APP_SUBTYPE_MAP(XX)) +#undef XX + +/* DATA subtype strings */ + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(data_subTypeStr_##subtype, #subtype) +PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::Data::subtype, &data_subTypeStr_##subtype}, +DEFINE_FSTR_MAP_LOCAL(dataSubTypeStrings, Partition::SubType::Data, FlashString, PARTITION_DATA_SUBTYPE_MAP(XX)) +#undef XX + +#define XX(subtype, value, desc) DEFINE_FSTR_LOCAL(data_subTypeStr_long_##subtype, desc) +PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + +#define XX(subtype, value, desc) {Partition::SubType::Data::subtype, &data_subTypeStr_long_##subtype}, +DEFINE_FSTR_MAP_LOCAL(longDataSubTypeStrings, Partition::SubType::Data, FlashString, PARTITION_DATA_SUBTYPE_MAP(XX)) +#undef XX + +} // namespace + +String toString(Partition::Type type, uint8_t subType) +{ + String s; + switch(type) { + case Partition::Type::app: + s = F("app/"); + if(auto v = appSubTypeStrings[Partition::SubType::App(subType)]) { + s += String(v); + } else { + s += subType; + } + break; + case Partition::Type::data: + s = F("data/"); + if(auto v = dataSubTypeStrings[Partition::SubType::Data(subType)]) { + s += String(v); + } else { + s += subType; + } + break; + + case Partition::Type::storage: + s = F("storage/"); + if(auto v = toString(Device::Type(subType))) { + s += v; + } else { + s += subType; + } + break; + + default: + s = unsigned(type); + s += '.'; + s += subType; + } + return s; +} + +String toLongString(Partition::Type type, uint8_t subType) +{ + switch(type) { + case Partition::Type::app: + if(auto v = longAppSubTypeStrings[Partition::SubType::App(subType)]) { + return F("App: ") + String(v); + } + break; + case Partition::Type::data: + if(auto v = longDataSubTypeStrings[Partition::SubType::Data(subType)]) { + return F("Data: ") + String(v); + } + break; + + case Partition::Type::storage: + if(auto s = toLongString(Device::Type(subType))) { + return F("Storage: ") + s; + } + break; + + default:; + // Unknown + } + + return toString(type, subType); +} + +namespace Storage +{ +String Partition::typeString() const +{ + return toString(type(), subType()); +} + +String Partition::longTypeString() const +{ + return toLongString(type(), subType()); +} + +bool Partition::verify(Partition::Type type, uint8_t subtype) const +{ + if(mPart == nullptr) { + debug_e("[Partition] invalid"); + return false; + } + + if(type != mPart->type) { + debug_e("[Partition] type mismatch, expected %u got %u", unsigned(type), unsigned(mPart->type)); + return false; + } + + if(mPart->subtype != subtype) { + debug_e("[Partition] subtype mismatch, expected %u got %u", subtype, mPart->subtype); + return false; + } + + return true; +} + +bool Partition::getDeviceAddress(uint32_t& address, size_t size) const +{ + if(mDevice == nullptr || mPart == nullptr) { + debug_e("[Partition] Invalid"); + return false; + } + + if(address >= mPart->size || (address + size - 1) >= mPart->size) { + debug_e("[Partition] Invalid range, address: 0x%08x, size: 0x%08x", address, size); + return false; + } + + // Storage partitions refer directly to the underlying device + if(type() != Partition::Type::storage) { + address += mPart->offset; + } + + return true; +} + +String Partition::getDeviceName() const +{ + return mDevice ? mDevice->getName() : nullptr; +} + +size_t Partition::getBlockSize() const +{ + return mDevice ? mDevice->getBlockSize() : 0; +} + +bool Partition::allowRead() +{ + if(mDevice == nullptr || mPart == nullptr) { + debug_e("[Partition] Invalid"); + return false; + } + + return true; +} + +bool Partition::allowWrite() +{ + if(!allowRead()) { + return false; + } + + if(mPart->flags[Flag::readOnly]) { + debug_e("[Partition] %s is read-only", mPart ? mPart->name.c_str() : "?"); + return false; + } + + return true; +} + +bool Partition::read(size_t offset, void* dst, size_t size) +{ + if(!allowRead()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->read(addr, dst, size) : false; +} + +bool Partition::write(size_t offset, const void* src, size_t size) +{ + if(!allowWrite()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->write(addr, src, size) : false; +} + +bool Partition::erase_range(size_t offset, size_t size) +{ + if(!allowWrite()) { + return false; + } + + uint32_t addr = offset; + if(!getDeviceAddress(addr, size)) { + return false; + } + + return mDevice ? mDevice->erase_range(addr, size) : false; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/PartitionStream.cpp b/Sming/Components/Storage/src/PartitionStream.cpp new file mode 100644 index 0000000000..86ad9b037c --- /dev/null +++ b/Sming/Components/Storage/src/PartitionStream.cpp @@ -0,0 +1,61 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionStream.cpp + * + ****/ + +#include "include/Storage/PartitionStream.h" + +namespace Storage +{ +uint16_t PartitionStream::readMemoryBlock(char* data, int bufSize) +{ + int len = std::min(bufSize, available()); + return partition.read(startOffset + readPos, data, len) ? len : 0; +} + +int PartitionStream::seekFrom(int offset, SeekOrigin origin) +{ + size_t newPos; + switch(origin) { + case SeekOrigin::Start: + newPos = 0; + break; + case SeekOrigin::Current: + newPos = readPos + offset; + break; + case SeekOrigin::End: + newPos = size + offset; + break; + default: + return -1; + } + + if(newPos > size) { + return -1; + } + + readPos = newPos; + return readPos; +} + +size_t PartitionStream::write(const uint8_t* data, size_t length) +{ + auto len = std::min(size - writePos, length); + if(len != 0) { + if(!partition.write(startOffset + writePos, data, len)) { + len = 0; + } else { + writePos += len; + } + } + + // Return amount actually written + return len; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/PartitionTable.cpp b/Sming/Components/Storage/src/PartitionTable.cpp new file mode 100644 index 0000000000..793cd061bf --- /dev/null +++ b/Sming/Components/Storage/src/PartitionTable.cpp @@ -0,0 +1,37 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionTable.cpp + * + ****/ + +#include "include/Storage/PartitionTable.h" +#include "include/Storage/partition_info.h" +#include + +namespace Storage +{ +void PartitionTable::load(const esp_partition_info_t* entry, unsigned count) +{ + if(count == 0) { + mEntries.reset(); + mCount = count; + return; + } + + mCount = count; + mEntries.reset(new Partition::Info[count]); + for(unsigned i = 0; i < count; ++i) { + auto& e = entry[i]; + // name may not be zero-terminated + char name[Partition::nameSize + 1]; + memcpy(name, e.name, Partition::nameSize); + name[Partition::nameSize] = '\0'; + mEntries.get()[i] = Partition::Info{name, e.type, e.subtype, e.offset, e.size, e.flags}; + } +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/ProgMem.cpp b/Sming/Components/Storage/src/ProgMem.cpp new file mode 100644 index 0000000000..8d95f594cc --- /dev/null +++ b/Sming/Components/Storage/src/ProgMem.cpp @@ -0,0 +1,35 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * ProgMem.cpp + * + ****/ + +#include "include/Storage/ProgMem.h" +#include + +namespace Storage +{ +ProgMem progMem; + +bool ProgMem::read(uint32_t address, void* dst, size_t size) +{ + size_t readCount = flashmem_read(dst, address, size); + return readCount == size; +} + +Partition ProgMem::createPartition(const String& name, const void* flashPtr, size_t size, Partition::Type type, + uint8_t subtype) +{ + auto addr = flashmem_get_address(flashPtr); + if(addr == 0) { + return Partition{}; + } + + return createPartition(name, type, subtype, addr, size, Partition::Flag::readOnly); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/SpiFlash.cpp b/Sming/Components/Storage/src/SpiFlash.cpp new file mode 100644 index 0000000000..a308d65c49 --- /dev/null +++ b/Sming/Components/Storage/src/SpiFlash.cpp @@ -0,0 +1,66 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SpiFlash.cpp + * + ****/ + +#include "include/Storage/SpiFlash.h" +#include "include/Storage/partition_info.h" +#include + +namespace Storage +{ +DEFINE_FSTR(FS_SPIFLASH, "spiFlash") +SpiFlash* spiFlash; + +String SpiFlash::getName() const +{ + return FS_SPIFLASH; +} + +size_t SpiFlash::getBlockSize() const +{ + return SPI_FLASH_SEC_SIZE; +} + +size_t SpiFlash::getSize() const +{ + return flashmem_get_size_bytes(); +} + +bool SpiFlash::read(uint32_t address, void* dst, size_t size) +{ + size_t readCount = flashmem_read(dst, address, size); + return readCount == size; +} + +bool SpiFlash::write(uint32_t address, const void* src, size_t size) +{ + size_t writeCount = flashmem_write(src, address, size); + return writeCount == size; +} + +bool SpiFlash::erase_range(uint32_t address, size_t size) +{ + if(address % SPI_FLASH_SEC_SIZE != 0 || size % SPI_FLASH_SEC_SIZE != 0) { + debug_e("[Partition] erase address/size misaligned: 0x%08x / 0x%08x", address, size); + return false; + } + + auto sec = address / SPI_FLASH_SEC_SIZE; + auto end = (address + size) / SPI_FLASH_SEC_SIZE; + while(sec < end) { + if(!flashmem_erase_sector(sec)) { + return false; + } + ++sec; + } + + return true; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/Storage.cpp b/Sming/Components/Storage/src/Storage.cpp new file mode 100644 index 0000000000..98b6c8ed18 --- /dev/null +++ b/Sming/Components/Storage/src/Storage.cpp @@ -0,0 +1,75 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * DeviceManager.cpp + * + ****/ + +#include "include/Storage.h" +#include "include/Storage/SpiFlash.h" +#include + +namespace Storage +{ +void initialize() +{ + if(spiFlash == nullptr) { + spiFlash = new SpiFlash; + registerDevice(spiFlash); + spiFlash->loadPartitions(PARTITION_TABLE_OFFSET); + } +} + +const Device::List getDevices() +{ + return Device::List(spiFlash); +} + +bool registerDevice(Device* device) +{ + if(device == nullptr) { + return false; + } + auto devname = device->getName(); + + Device::List devices(spiFlash); + auto it = std::find(devices.begin(), devices.end(), devname); + if(!it) { + devices.add(device); + device->loadPartitions(*spiFlash, PARTITION_TABLE_OFFSET); + debug_i("[Storage] Device '%s' registered", devname.c_str()); + } else if(*it != *device) { + debug_e("[Storage] Another device is already registered with name '%s'", devname.c_str()); + return false; + } + + return true; +} + +bool unRegisterDevice(Device* device) +{ + return Device::List(spiFlash).remove(device); +} + +Device* findDevice(const String& name) +{ + Device::List devices(spiFlash); + return std::find(devices.begin(), devices.end(), name); +} + +Partition findPartition(const String& name) +{ + for(auto& dev : getDevices()) { + auto part = dev.partitions().find(name); + if(part) { + return part; + } + } + + return Partition{}; +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/SysMem.cpp b/Sming/Components/Storage/src/SysMem.cpp new file mode 100644 index 0000000000..5281259f62 --- /dev/null +++ b/Sming/Components/Storage/src/SysMem.cpp @@ -0,0 +1,25 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SysMem.cpp + * + ****/ + +#include "include/Storage/SysMem.h" +#include + +namespace Storage +{ +SysMem sysMem; + +Partition SysMem::createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, + uint8_t subtype) +{ + return createPartition(name, type, subtype, reinterpret_cast(fstr.data()), fstr.size(), + Partition::Flag::readOnly); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage.h b/Sming/Components/Storage/src/include/Storage.h new file mode 100644 index 0000000000..81977f4d85 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage.h @@ -0,0 +1,62 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Storage.h + * + ****/ +#pragma once + +#include "Storage/Device.h" + +namespace Storage +{ +/** + * @brief Called early in the startup phase + */ +void initialize(); + +/** + * @brief Get read-only reference to device list + */ +const Device::List getDevices(); + +/** + * @brief Register a storage device + * @retval bool true on success, false if another device already registered with same name + */ +bool registerDevice(Device* device); + +/** + * @brief Unregister a storage device + * + * Use extreme care: behaviour is unpredictable if partitions are in use + */ +bool unRegisterDevice(Device* device); + +/** + * @brief Find a registered device + */ +Device* findDevice(const String& name); + +/** + * @brief Find the first partition matching the given name + */ +Partition findPartition(const String& name); + +/** + * @brief Find partitions of the given type + */ +inline Iterator findPartition(Partition::Type type = Partition::Type::any, uint8_t subType = Partition::SubType::any) +{ + return Iterator(type, subType); +} + +template Iterator findPartition(T subType) +{ + return Iterator(Partition::Type(T::partitionType), uint8_t(subType)); +} + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/CustomDevice.h b/Sming/Components/Storage/src/include/Storage/CustomDevice.h new file mode 100644 index 0000000000..82222f2667 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/CustomDevice.h @@ -0,0 +1,35 @@ +/* + * CustomDevice.h + */ + +#pragma once + +#include "Device.h" + +namespace Storage +{ +/** + * @brief Class to support dynamic partitions + * + * Call `createPartition` to add partitions up to a maximum of 16 entries. + */ +class CustomDevice : public Device +{ +public: + Partition createPartition(const Partition::Info& info); + + Partition createPartition(const String& name, Partition::Type type, uint8_t subtype, uint32_t offset, size_t size, + Partition::Flags flags = 0) + { + return createPartition(Partition::Info{name, type, subtype, offset, size, flags}); + } + + template + Partition createPartition(const String& name, SubType subtype, uint32_t offset, size_t size, + Partition::Flags flags = 0) + { + return createPartition(name, Partition::Type(SubType::partitionType), uint8_t(subtype), offset, size, flags); + } +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/Device.h b/Sming/Components/Storage/src/include/Storage/Device.h new file mode 100644 index 0000000000..cf54588b19 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Device.h @@ -0,0 +1,140 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Device.h - external storage device API + * + ****/ +#pragma once + +#include +#include "ObjectList.h" +#include "PartitionTable.h" + +#define STORAGE_TYPE_MAP(XX) \ + XX(unknown, 0x00, "Other storage device") \ + XX(flash, 0x01, "SPI flash") \ + XX(spiram, 0x02, "SPI RAM") \ + XX(sdcard, 0x03, "SD Card") \ + XX(disk, 0x04, "Physical disk") \ + XX(file, 0x05, "Backing file on separate filesystem") \ + XX(sysmem, 0x06, "System Memory") + +namespace Storage +{ +class SpiFlash; + +/** + * @brief Represents a storage device (e.g. flash memory) + */ +class Device : public ObjectTemplate +{ +public: + using List = ObjectListTemplate; + using OwnedList = OwnedObjectListTemplate; + + /** + * @brief Storage type + */ + enum class Type : uint8_t { +#define XX(type, value, desc) type = value, + STORAGE_TYPE_MAP(XX) +#undef XX + }; + + Device() : mPartitions(*this) + { + } + + ~Device(); + + bool operator==(const String& name) const + { + return getName() == name; + } + + PartitionTable& partitions() + { + return mPartitions; + } + + const PartitionTable& partitions() const + { + return mPartitions; + } + + /** + * @brief Load partition table entries + * @tableOffset Location of partition table to read + * @retval bool true on success, false on failure + */ + bool loadPartitions(uint32_t tableOffset) + { + return loadPartitions(*this, tableOffset); + } + + /** + * @brief Load partition table entries from another table + * @param source Device to load entries from + * @tableOffset Location of partition table to read + * @retval bool true on success, false on failure + */ + bool loadPartitions(Device& source, uint32_t tableOffset); + + /** + * @brief Obtain unique device name + */ + virtual String getName() const = 0; + + /** + * @brief Obtain smallest allocation unit for erase operations + */ + virtual size_t getBlockSize() const = 0; + + /** + * @brief Obtain addressable size of this device + * @retval size_t Must be at least as large as the value declared in the partition table + */ + virtual size_t getSize() const = 0; + + /** + * @brief Obtain device type + */ + virtual Type getType() const = 0; + + /** + * @brief Read data from the storage device + * @param address Where to start reading + * @param dst Buffer to store data + * @param size Size of data to be read, in bytes. + * @retval bool true on success, false on error + */ + virtual bool read(uint32_t address, void* dst, size_t size) = 0; + + /** + * @brief Write data to the storage device + * @param address Where to start writing + * @param src Data to write + * @param size Size of data to be written, in bytes. + * @retval bool true on success, false on error + */ + virtual bool write(uint32_t address, const void* src, size_t size) = 0; + + /** + * @brief Erase a region of storage in preparation for writing + * @param address Where to start erasing + * @param size Size of region to erase, in bytes + * @retval bool true on success, false on error + */ + virtual bool erase_range(uint32_t address, size_t size) = 0; + +protected: + PartitionTable mPartitions; +}; + +} // namespace Storage + +String toString(Storage::Device::Type type); +String toLongString(Storage::Device::Type type); diff --git a/Sming/Components/Storage/src/include/Storage/Iterator.h b/Sming/Components/Storage/src/include/Storage/Iterator.h new file mode 100644 index 0000000000..dcd3b7ca88 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Iterator.h @@ -0,0 +1,78 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Iterator.h + * + ****/ +#pragma once + +#include "Partition.h" + +namespace Storage +{ +class Device; + +class Iterator : public std::iterator +{ +public: + Iterator(Device& device, uint8_t partitionIndex); + + Iterator(Device& device, Partition::Type type, uint8_t subtype) : mSearch{&device, type, subtype} + { + mDevice = &device; + next(); + } + + Iterator(Partition::Type type, uint8_t subtype); + + explicit operator bool() const + { + return (mDevice != nullptr) && (mPos > beforeStart) && (mPos < afterEnd); + } + + Iterator operator++(int) + { + auto result = *this; + next(); + return result; + } + + Iterator& operator++() + { + next(); + return *this; + } + + bool operator==(const Iterator& other) const + { + return (mDevice == other.mDevice) && (mPos == other.mPos); + } + + bool operator!=(const Iterator& other) const + { + return !operator==(other); + } + + Partition operator*() const; + +private: + static constexpr int8_t beforeStart{-1}; + static constexpr int8_t afterEnd{0x7f}; + + bool seek(uint8_t pos); + bool next(); + + struct Search { + Device* device; + Partition::Type type; + uint8_t subType; + }; + Search mSearch{}; + Device* mDevice{nullptr}; + int8_t mPos{beforeStart}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/Object.h b/Sming/Components/Storage/src/include/Storage/Object.h new file mode 100644 index 0000000000..0d6914f643 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Object.h @@ -0,0 +1,144 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Object.h - Base Storage object definition + * + ****/ +#pragma once + +#include +#include + +namespace Storage +{ +class ObjectList; + +class Object +{ +public: + virtual ~Object() + { + } + + virtual Object* next() const + { + return mNext; + } + + Object* getNext() const + { + return mNext; + } + + bool operator==(const Object& other) const + { + return this == &other; + } + + bool operator!=(const Object& other) const + { + return this != &other; + } + +private: + friend class ObjectList; + Object* mNext{nullptr}; +}; + +/** + * @brief Base class template for linked items with type casting + */ +template class ObjectTemplate : public Object +{ +public: + template + class IteratorTemplate : public std::iterator + { + public: + IteratorTemplate(TPtr x) : mObject(x) + { + } + + IteratorTemplate(TRef& x) : mObject(&x) + { + } + + IteratorTemplate(const IteratorTemplate& other) : mObject(other.mObject) + { + } + + IteratorTemplate& operator++() + { + mObject = mObject->getNext(); + return *this; + } + + IteratorTemplate operator++(int) + { + Iterator tmp(*this); + operator++(); + return tmp; + } + + bool operator==(const IteratorTemplate& rhs) const + { + return mObject == rhs.mObject; + } + + bool operator!=(const IteratorTemplate& rhs) const + { + return mObject != rhs.mObject; + } + + TRef operator*() + { + return *mObject; + } + + TPtr operator->() + { + return mObject; + } + + operator TPtr() + { + return mObject; + } + + private: + TPtr mObject; + }; + + using Iterator = IteratorTemplate; + using ConstIterator = IteratorTemplate; + + ObjectType* getNext() const + { + return reinterpret_cast(this->next()); + } + + Iterator begin() const + { + return Iterator(this); + } + + Iterator end() const + { + return Iterator(nullptr); + } + + Iterator cbegin() const + { + return ConstIterator(this); + } + + Iterator cend() const + { + return ConstIterator(nullptr); + } +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/ObjectList.h b/Sming/Components/Storage/src/include/Storage/ObjectList.h new file mode 100644 index 0000000000..da015dd0d8 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/ObjectList.h @@ -0,0 +1,137 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Object.h - Base Storage object definition + * + ****/ +#pragma once + +#include "Object.h" +#include + +namespace Storage +{ +/** + * @brief Singly-linked list of objects + * @note We don't own the items, just keep references to them + */ +class ObjectList +{ +public: + ObjectList() + { + } + + ObjectList(Object* object) : mHead(object) + { + } + + bool add(Object* object); + + bool add(const Object* object) + { + return add(const_cast(object)); + } + + bool remove(Object* object); + + void clear() + { + mHead = nullptr; + } + + Object* head() + { + return mHead; + } + + const Object* head() const + { + return mHead; + } + + bool isEmpty() const + { + return mHead == nullptr; + } + +protected: + Object* mHead{nullptr}; +}; + +template class ObjectListTemplate : public ObjectList +{ +public: + ObjectListTemplate() = default; + + ObjectListTemplate(ObjectType* object) : ObjectList(object) + { + } + + ObjectType* head() + { + return reinterpret_cast(mHead); + } + + const ObjectType* head() const + { + return reinterpret_cast(mHead); + } + + typename ObjectType::Iterator begin() + { + return head(); + } + + typename ObjectType::Iterator end() + { + return nullptr; + } + + typename ObjectType::ConstIterator begin() const + { + return head(); + } + + typename ObjectType::ConstIterator end() const + { + return nullptr; + } + + size_t count() const + { + return std::count(begin(), end(), true); + } + + bool contains(const ObjectType& object) const + { + return std::find(begin(), end(), object); + } +}; + +/** + * @brief Class template for singly-linked list of objects + * @note We own the objects so are responsible for destroying them when removed + */ +template class OwnedObjectListTemplate : public ObjectListTemplate +{ +public: + bool remove(ObjectType* object) + { + bool res = ObjectList::remove(object); + delete object; + return res; + } + + void clear() + { + while(remove(this->head())) { + // + } + } +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/Partition.h b/Sming/Components/Storage/src/include/Storage/Partition.h new file mode 100644 index 0000000000..5428e1e2a7 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/Partition.h @@ -0,0 +1,376 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * Partition.h - C++ wrapper for universal partition table support + * + * Original license for IDF code: + * + * Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + ****/ +#pragma once + +#include +#include +#include +#include + +#define PARTITION_APP_SUBTYPE_MAP(XX) \ + XX(factory, 0x00, "Factory application") \ + XX(ota0, 0x10, "OTA #0") \ + XX(ota1, 0x11, "OTA #1") \ + XX(ota2, 0x12, "OTA #2") \ + XX(ota3, 0x13, "OTA #3") \ + XX(ota4, 0x14, "OTA #4") \ + XX(ota5, 0x15, "OTA #5") \ + XX(ota6, 0x16, "OTA #6") \ + XX(ota7, 0x17, "OTA #7") \ + XX(ota8, 0x18, "OTA #8") \ + XX(ota9, 0x19, "OTA #9") \ + XX(ota10, 0x1a, "OTA #10") \ + XX(ota11, 0x1b, "OTA #11") \ + XX(ota12, 0x1c, "OTA #12") \ + XX(ota13, 0x1d, "OTA #13") \ + XX(ota14, 0x1e, "OTA #14") \ + XX(ota15, 0x1f, "OTA #15") \ + XX(test, 0x20, "Test application") + +#define PARTITION_DATA_SUBTYPE_MAP(XX) \ + XX(ota, 0x00, "OTA selection") \ + XX(phy, 0x01, "PHY init data") \ + XX(nvs, 0x02, "NVS") \ + XX(coreDump, 0x03, "Core Dump data") \ + XX(nvsKeys, 0x04, "NVS key information") \ + XX(eFuseEm, 0x05, "eFuse emulation") \ + XX(sysParam, 0x40, "System Parameters") \ + XX(espHttpd, 0x80, "ESPHTTPD") \ + XX(fat, 0x81, "FAT") \ + XX(spiffs, 0x82, "SPIFFS") + +namespace Storage +{ +class Device; +class PartitionTable; +struct esp_partition_info_t; + +/** + * @brief Represents a flash partition + */ +class Partition +{ +public: + enum class Type : uint8_t { + app = 0x00, + data = 0x01, + storage = 0x02, + userMin = 0x40, + userMax = 0xFE, + invalid = 0xff, + any = 0xff, + }; + + struct SubType { + static constexpr uint8_t any{0xff}; + static constexpr uint8_t invalid{0xff}; + + /** + * @brief Application partition type + */ + enum class App : uint8_t { + partitionType = uint8_t(Type::app), +#define XX(type, value, desc) type = value, + PARTITION_APP_SUBTYPE_MAP(XX) +#undef XX + ota_min = ota0, + ota_max = ota15, + any = 0xff + }; + + /** + * @brief Data partition type + */ + enum class Data : uint8_t { + partitionType = uint8_t(Type::data), +#define XX(subtype, value, desc) subtype = value, + PARTITION_DATA_SUBTYPE_MAP(XX) +#undef XX + any = 0xff + }; + }; + + enum class Flag { + encrypted = 0, + readOnly = 31, ///< Write/erase prohibited + }; + + static constexpr size_t nameSize{16}; + using Name = char[nameSize]; + using Flags = BitSet; + + /** + * @brief Partition information + */ + struct Info { + CString name; + uint32_t offset{0}; + uint32_t size{0}; + Type type{Type::invalid}; + uint8_t subtype{SubType::invalid}; + Flags flags; + + Info() + { + } + + Info(const String& name, Type type, uint8_t subtype, uint32_t offset, uint32_t size, Flags flags) + : name(name), offset(offset), size(size), type(type), subtype(subtype), flags(flags) + { + } + }; + + Partition() + { + } + + Partition(const Partition& other) : mDevice(other.mDevice), mPart(other.mPart) + { + } + + Partition(Device& device, const Info& info) : mDevice(&device), mPart(&info) + { + } + + /** + * @name Confirm partition is of the expected type + * @param type Expected partition type + * @param subtype Expected partition sub-type + * @retval bool true if type is OK, false if not. + * Logs debug messages on failure. + * @{ + */ + bool verify(Type type, uint8_t subtype) const; + + bool verify(uint8_t type, uint8_t subtype) const + { + return verify(Type(type), subtype); + } + + template bool verify(T subType) const + { + return verify(Type(T::partitionType), uint8_t(subType)); + } + + /** @} */ + + /** + * @brief Convenience function to get SubType value for the i-th OTA partition + */ + static inline SubType::App apptypeOta(uint8_t i) + { + auto subtype = SubType::App(uint8_t(SubType::App::ota_min) + i); + assert(subtype >= SubType::App::ota_min && subtype <= SubType::App::ota_max); + return subtype; + } + + explicit operator bool() const + { + return mDevice != nullptr && mPart != nullptr; + } + + /** + * @brief Read data from the partition + * @param offset Where to start reading, relative to start of partition + * @param dst Buffer to store data + * @param size Size of data to be read, in bytes. + * @retval bool true on success, false on error + */ + bool read(size_t offset, void* dst, size_t size); + + template typename std::enable_if::value, bool>::type read(size_t offset, T& value) + { + return read(offset, &value, sizeof(value)); + } + + /** + * @brief Write data to the partition + * @param offset Where to start writing, relative to start of partition + * @param src Data to write + * @param size Size of data to be written, in bytes. + * @retval bool true on success, false on error + * @note Flash region must be erased first + */ + bool write(size_t offset, const void* src, size_t size); + + /** + * @brief Erase part of the partition + * @param offset Where to start erasing, relative to start of partition + * @param size Size of region to erase, in bytes + * @retval bool true on success, false on error + * @note Both offset and size must be aligned to flash sector size (4Kbytes) + */ + bool erase_range(size_t offset, size_t size); + + /** + * @brief Obtain partition type + */ + Partition::Type type() const + { + return mPart ? Partition::Type(mPart->type) : Type::invalid; + } + + /** + * @brief Obtain partition sub-type + */ + uint8_t subType() const + { + return mPart ? mPart->subtype : SubType::invalid; + } + + /** + * @brief Obtain partition starting address + * @param uint32_t Device address + */ + uint32_t address() const + { + return (mPart && mPart->type != Partition::Type::storage) ? mPart->offset : 0; + } + + /** + * @brief Obtain address of last byte in this this partition + * @param uint32_t Device address + */ + uint32_t lastAddress() const + { + return mPart ? (mPart->offset + mPart->size - 1) : 0; + } + + /** + * @brief Obtain partition size + * @retval uint32_t Size in bytes + */ + uint32_t size() const + { + return mPart ? mPart->size : 0; + } + + /** + * @brief Get partition name + */ + String name() const + { + return mPart ? mPart->name.c_str() : nullptr; + } + + /** + * @brief Get partition flags + */ + Flags flags() const + { + return mPart ? mPart->flags : 0; + } + + /** + * @brief Check state of partition `encrypted` flag + */ + bool isEncrypted() const + { + return flags()[Flag::encrypted]; + } + + /** + * @brief Check state of partition `readOnly` flag + */ + bool isReadOnly() const + { + return mPart ? mPart->flags[Flag::readOnly] : true; + } + + /** + * @name Get partition type expressed as a string + * @{ + */ + String typeString() const; + String longTypeString() const; + /** @} */ + + /** + * @brief Get corresponding storage device address for a given partition offset + * @param address IN: Zero-based offset within partition, OUT: Device address + * @param size Size of data to be accessed + * @retval bool true on success, false on failure + * Fails if the given offset/size combination is out of range, or the partition is undefined. + */ + bool getDeviceAddress(uint32_t& address, size_t size) const; + + /** + * @brief Get name of storage device for this partition + * @retval String + */ + String getDeviceName() const; + + /** + * @brief Get storage device containing this partition + * @retval Device* null if device isn't registered + */ + Device* getDevice() const + { + return mDevice; + } + + /** + * @brief Determine if given address contained within this partition + */ + bool contains(uint32_t addr) const + { + return mPart ? (addr >= mPart->offset && addr <= lastAddress()) : false; + } + + bool operator==(const Partition& other) const + { + return this == &other; + } + + bool operator==(const char* name) const + { + return mPart ? mPart->name.equals(name) : false; + } + + bool operator==(const String& name) const + { + return mPart ? mPart->name.equals(name) : false; + } + + /** + * @brief Obtain smallest allocation unit for erase operations + */ + size_t getBlockSize() const; + +protected: + Device* mDevice{nullptr}; + const Info* mPart{nullptr}; + +private: + bool allowRead(); + bool allowWrite(); +}; + +} // namespace Storage + +String toString(Storage::Partition::Type type, uint8_t subType); +String toLongString(Storage::Partition::Type type, uint8_t subType); diff --git a/Sming/Components/Storage/src/include/Storage/PartitionStream.h b/Sming/Components/Storage/src/include/Storage/PartitionStream.h new file mode 100644 index 0000000000..93debaeaff --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/PartitionStream.h @@ -0,0 +1,61 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionStream.h + * + ****/ + +#pragma once + +#include +#include "Partition.h" + +namespace Storage +{ +/** + * @brief Stream operating directory on a Storage partition + * + * To support write operations, the target region must be erased first. + * + * @ingroup stream + */ +class PartitionStream : public ReadWriteStream +{ +public: + PartitionStream(Partition partition, uint32_t offset, size_t size) + : partition(partition), startOffset(offset), size(size) + { + } + + PartitionStream(Partition partition) : partition(partition), startOffset(0), size(partition.size()) + { + } + + int available() override + { + return size - readPos; + } + + uint16_t readMemoryBlock(char* data, int bufSize) override; + + int seekFrom(int offset, SeekOrigin origin) override; + + size_t write(const uint8_t* buffer, size_t size) override; + + bool isFinished() override + { + return available() <= 0; + } + +private: + Partition partition; + uint32_t startOffset; + size_t size; + uint32_t writePos{0}; + uint32_t readPos{0}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/PartitionTable.h b/Sming/Components/Storage/src/include/Storage/PartitionTable.h new file mode 100644 index 0000000000..1dd4ee7b16 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/PartitionTable.h @@ -0,0 +1,92 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * PartitionTable.h + * + ****/ +#pragma once + +#include "Partition.h" +#include "Iterator.h" + +namespace Storage +{ +// Used Partition table entries cached in RAM, initialised on first request +class PartitionTable +{ +public: + PartitionTable(Device& device) : mDevice(device) + { + } + + /** + * @name Partition search + * @{ + * + * @brief Find partitions based on one or more parameters + * @param type Partition type + * @param subtype Partition sub-type + * @retval Iterator Forward-iterator for matching partitions + */ + Iterator find(Partition::Type type = Partition::Type::any, uint8_t subType = Partition::SubType::any) const + { + return Iterator(mDevice, type, subType); + } + + template Iterator find(T subType) const + { + return find(Partition::Type(T::partitionType), uint8_t(subType)); + } + + /** @} */ + + /** + * @brief Find partition by name + * @param Name to search for, case-sensitive + * @retval Partition + * + * Names are unique so at most only one match + */ + Partition find(const String& name) const + { + return *std::find(begin(), end(), name); + } + + Iterator begin() const + { + return Iterator(mDevice, 0); + } + + Iterator end() const + { + return Iterator(mDevice, mCount); + } + + uint8_t count() const + { + return mCount; + } + + Device& device() const + { + return mDevice; + } + + Partition operator[](unsigned index) const + { + return (index < mCount) ? Partition(mDevice, mEntries.get()[index]) : Partition(); + } + +protected: + friend Device; + void load(const esp_partition_info_t* entry, unsigned count); + + Device& mDevice; + std::unique_ptr mEntries; + uint8_t mCount{0}; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/ProgMem.h b/Sming/Components/Storage/src/include/Storage/ProgMem.h new file mode 100644 index 0000000000..d8758244d0 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/ProgMem.h @@ -0,0 +1,89 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * ProgMem.h + * + ****/ +#pragma once + +#include "CustomDevice.h" + +namespace Storage +{ +/** + * @brief Storage device to access PROGMEM using flash API + */ +class ProgMem : public CustomDevice +{ +public: + String getName() const override + { + return F("ProgMem"); + } + + size_t getBlockSize() const override + { + return sizeof(uint32_t); + } + + size_t getSize() const override + { + return 0x80000000; + } + + Type getType() const override + { + return Type::flash; + } + + bool read(uint32_t address, void* dst, size_t size) override; + + bool write(uint32_t address, const void* src, size_t size) override + { + return false; + } + + bool erase_range(uint32_t address, size_t size) override + { + return false; + } + + using CustomDevice::createPartition; + + /** + * @brief Create partition for PROGMEM data access + * @param name Name for partition + * @param flashPtr PROGMEM pointer + * @param size Size of PROGMEM data + * @param type Partition type + * @param subtype Partition sub-type + * @retval Partition Invalid if data is not progmem + */ + Partition createPartition(const String& name, const void* flashPtr, size_t size, Partition::Type type, + uint8_t subtype); + + template Partition createPartition(const String& name, const void* flashPtr, size_t size, T subType) + { + return createPartition(name, flashPtr, size, Partition::Type(T::partitionType), uint8_t(subType)); + } + + /** + * @brief Create partition for FlashString data access + */ + Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, uint8_t subtype) + { + return createPartition(name, fstr.data(), fstr.size(), type, subtype); + } + + template Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, T subType) + { + return createPartition(name, fstr, Partition::Type(T::partitionType), uint8_t(subType)); + } +}; + +extern ProgMem progMem; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/SpiFlash.h b/Sming/Components/Storage/src/include/Storage/SpiFlash.h new file mode 100644 index 0000000000..a5f0319362 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/SpiFlash.h @@ -0,0 +1,38 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SpiFlash.h + * + ****/ +#pragma once + +#include "Device.h" + +namespace Storage +{ +extern SpiFlash* spiFlash; + +/** + * @brief Main flash storage device + */ +class SpiFlash : public Device +{ +public: + String getName() const override; + size_t getBlockSize() const override; + size_t getSize() const override; + + Type getType() const override + { + return Type::flash; + } + + bool read(uint32_t address, void* dst, size_t size) override; + bool write(uint32_t address, const void* src, size_t size) override; + bool erase_range(uint32_t address, size_t size) override; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/StreamDevice.h b/Sming/Components/Storage/src/include/Storage/StreamDevice.h new file mode 100644 index 0000000000..18ca2d64cf --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/StreamDevice.h @@ -0,0 +1,67 @@ +/* + * StreamDevice.h + */ + +#include "CustomDevice.h" +#include + +namespace Storage +{ +/** + * @brief Read-only partition on a stream object + * @note Writes not possible as streams always append data, cannot do random writes + */ +class StreamDevice : public CustomDevice +{ +public: + StreamDevice(IDataSourceStream* stream, size_t size) : CustomDevice(nameOf(stream), size), mStream(stream) + { + } + + StreamDevice(IDataSourceStream* stream) : StreamDevice(stream, size_t(stream->available())) + { + } + + static String nameOf(IDataSourceStream* stream) + { + String s; + if(stream != nullptr) { + s = stream->getName(); + } + if(!s) { + s = F("stream_") + String(uint32_t(stream), HEX); + } + return s; + } + + Type getType() const override + { + return Type::stream; + } + + bool read(uint32_t address, void* buffer, size_t len) override + { + if(mStream == nullptr) { + return false; + } + if(mStream->seekFrom(address, SeekOrigin::Start) != int(address)) { + return false; + } + return mStream->readBytes(static_cast(buffer), len) == len; + } + + bool write(uint32_t address, const void* data, size_t len) override + { + return false; + } + + bool erase_range(uint32_t address, size_t len) override + { + return false; + } + +private: + std::unique_ptr mStream; +}; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/SysMem.h b/Sming/Components/Storage/src/include/Storage/SysMem.h new file mode 100644 index 0000000000..9f49a3fa3e --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/SysMem.h @@ -0,0 +1,88 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * SysMem.h + * + ****/ + +#pragma once + +#include "CustomDevice.h" + +namespace Storage +{ +/** + * @brief Storage device to access system memory, e.g. RAM + */ +class SysMem : public CustomDevice +{ +public: + String getName() const override + { + return F("SysMem"); + } + + size_t getBlockSize() const override + { + return sizeof(uint32_t); + } + + size_t getSize() const override + { + return 0x80000000; + } + + Type getType() const override + { + return Type::sysmem; + } + + bool read(uint32_t address, void* buffer, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + memcpy_P(buffer, reinterpret_cast(address), len); + } else { + memcpy(buffer, reinterpret_cast(address), len); + } + return true; + } + + bool write(uint32_t address, const void* data, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + return false; + } + + memcpy(reinterpret_cast(address), data, len); + return true; + } + + bool erase_range(uint32_t address, size_t len) override + { + if(isFlashPtr(reinterpret_cast(address))) { + return false; + } + + memset(&address, 0xFF, len); + return true; + } + + using CustomDevice::createPartition; + + /** + * @brief Create partition for FlashString data access + */ + Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, Partition::Type type, uint8_t subtype); + + template Partition createPartition(const String& name, const FSTR::ObjectBase& fstr, T subType) + { + return createPartition(name, fstr, Partition::Type(T::partitionType), uint8_t(subType)); + } +}; + +extern SysMem sysMem; + +} // namespace Storage diff --git a/Sming/Components/Storage/src/include/Storage/partition_info.h b/Sming/Components/Storage/src/include/Storage/partition_info.h new file mode 100644 index 0000000000..a4b54c7f62 --- /dev/null +++ b/Sming/Components/Storage/src/include/Storage/partition_info.h @@ -0,0 +1,32 @@ +/**** + * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. + * Created 2015 by Skurydin Alexey + * http://github.com/SmingHub/Sming + * All files of the Sming Core are provided under the LGPL v3 license. + * + * partition_info.h + * + ****/ + +#include "Partition.h" + +namespace Storage +{ +/** + * @brief Internal structure describing the binary layout of a partition table entry. + */ +struct esp_partition_info_t { + uint16_t magic; ///< Fixed value to identify valid entry, appears as 0xFFFF at end of table + Partition::Type type; ///< Main type of partition + uint8_t subtype; ///< Sub-type for partition (interpretation dependent upon type) + uint32_t offset; ///< Start offset + uint32_t size; ///< Size of partition in bytes + Storage::Partition::Name name; ///< Unique identifer for entry + Storage::Partition::Flags flags; ///< Various option flags +}; + +constexpr uint16_t ESP_PARTITION_MAGIC{0x50AA}; ///< Identifies a valid partition +constexpr uint16_t ESP_PARTITION_MAGIC_MD5{0xEBEB}; ///< Identifies an MD5 hash block +constexpr size_t ESP_PARTITION_TABLE_MAX_LEN{0xC00}; // Maximum length of partition table data + +} // namespace Storage diff --git a/Sming/Core/SmingCore.h b/Sming/Core/SmingCore.h index 402665df6c..7420d1ce26 100644 --- a/Sming/Core/SmingCore.h +++ b/Sming/Core/SmingCore.h @@ -57,3 +57,5 @@ #include "DateTime.h" #include "fatfs/ff.h" + +#include diff --git a/Sming/component.mk b/Sming/component.mk index 62a9f18a22..05acf63028 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -13,6 +13,7 @@ COMPONENT_INCDIRS := \ . COMPONENT_DEPENDS := \ + Storage \ sming-arch \ FlashString \ spiffs \ diff --git a/Sming/project.mk b/Sming/project.mk index ec9440cbc1..bc3f64b678 100644 --- a/Sming/project.mk +++ b/Sming/project.mk @@ -55,6 +55,7 @@ CACHE_VARS := # Use PROJECT_DIR to identify the project source directory, from where this makefile must be included DEBUG_VARS += PROJECT_DIR PROJECT_DIR := $(CURDIR) +export PROJECT_DIR ifeq ($(MAKELEVEL),0) $(info ) From 1d3eeaa51c3f3b1966cd5efda68ecd9cc2d51b3b Mon Sep 17 00:00:00 2001 From: mikee47 Date: Tue, 8 Dec 2020 08:16:04 +0000 Subject: [PATCH 11/36] Use Partition API for rBoot, esp8266 startup code and SPIFFS * Remove `spiffs_mount_manual`, `spiffs_format_manual` and `spiffs_get_storage_config` - incompatible with partitions * Add overrides for `spiffs_mount(Partition&)` and `spiffs_format(Partition&)` --- .../Esp32/Components/esp32/src/startup.cpp | 2 + Sming/Arch/Esp32/Components/esp_spiffs/.cs | 0 .../Esp32/Components/esp_spiffs/README.rst | 4 - .../Esp32/Components/esp_spiffs/component.mk | 1 - .../Components/esp_spiffs/spiffs_config.c | 37 ------ .../Esp32/Components/sming-arch/component.mk | 1 - Sming/Arch/Esp32/app.mk | 28 ---- .../Esp8266/Components/esp8266/component.mk | 7 +- .../Esp8266/Components/esp8266/startup.cpp | 68 ++-------- Sming/Arch/Esp8266/app.mk | 2 +- .../Arch/Host/Components/hostlib/startup.cpp | 3 + Sming/Arch/Host/app.mk | 4 +- Sming/Components/esptool/component.mk | 1 - .../rboot/appcode/rboot-overrides.c | 43 ------ Sming/Components/rboot/component.mk | 20 ++- .../{appcode => include}/rboot-integration.h | 0 .../rboot/{host => src/Arch/Host}/rboot.cpp | 6 +- Sming/Components/spiffs/README.rst | 2 +- Sming/Components/spiffs/spiffs_config.h | 4 +- Sming/Components/spiffs/spiffs_sming.cpp | 122 ++++++++++-------- Sming/Components/spiffs/spiffs_sming.h | 45 ++----- Sming/Components/spiffs/spiffy/spiffy.c | 8 +- tests/HostTests/modules/Spiffs.cpp | 4 +- 23 files changed, 130 insertions(+), 282 deletions(-) delete mode 100644 Sming/Arch/Esp32/Components/esp_spiffs/.cs delete mode 100644 Sming/Arch/Esp32/Components/esp_spiffs/README.rst delete mode 100644 Sming/Arch/Esp32/Components/esp_spiffs/component.mk delete mode 100644 Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c delete mode 100644 Sming/Components/rboot/appcode/rboot-overrides.c rename Sming/Components/rboot/{appcode => include}/rboot-integration.h (100%) rename Sming/Components/rboot/{host => src/Arch/Host}/rboot.cpp (89%) diff --git a/Sming/Arch/Esp32/Components/esp32/src/startup.cpp b/Sming/Arch/Esp32/Components/esp32/src/startup.cpp index 602dd46c13..cdb1ff521b 100644 --- a/Sming/Arch/Esp32/Components/esp32/src/startup.cpp +++ b/Sming/Arch/Esp32/Components/esp32/src/startup.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #ifndef ESP32_STACK_SIZE #define ESP32_STACK_SIZE 16384U @@ -63,6 +64,7 @@ void main(void*) esp_init_flash(); esp_init_wifi(); ets_init_tasks(); + Storage::initialize(); System.initialize(); init(); diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/.cs b/Sming/Arch/Esp32/Components/esp_spiffs/.cs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/README.rst b/Sming/Arch/Esp32/Components/esp_spiffs/README.rst deleted file mode 100644 index 9869b71f45..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -Esp32 SPIFFS -============ - -SPIFFS implementation for ESP32 devices. diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/component.mk b/Sming/Arch/Esp32/Components/esp_spiffs/component.mk deleted file mode 100644 index 12ce0a9792..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/component.mk +++ /dev/null @@ -1 +0,0 @@ -COMPONENT_DEPENDS := spiffs diff --git a/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c b/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c deleted file mode 100644 index c8e5f095e9..0000000000 --- a/Sming/Arch/Esp32/Components/esp_spiffs/spiffs_config.c +++ /dev/null @@ -1,37 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * rboot-overrides.c - * - ****/ - -#include "spiffs_sming.h" -#include -#include - -/* - * rBoot uses different spiffs organization and we need to override this method - * during application compile time in order to make automatic - * mounting with `spiffs_mount()` work as expected. - */ -spiffs_config spiffs_get_storage_config() -{ - spiffs_config cfg = {0}; - - const esp_partition_t* partition = - esp_partition_find_first(ESP_PARTITION_TYPE_DATA, ESP_PARTITION_SUBTYPE_DATA_SPIFFS, NULL); - - if(partition == NULL) { - debug_w("No SPIFFS partition registered"); - } else { - cfg.phys_addr = partition->address; - cfg.phys_size = partition->size; - debug_w("SPIFFS partition found at 0x%08x, size 0x%08x", cfg.phys_addr, cfg.phys_size); - // TODO: Check partition->flash_chip is valid? - } - - return cfg; -} diff --git a/Sming/Arch/Esp32/Components/sming-arch/component.mk b/Sming/Arch/Esp32/Components/sming-arch/component.mk index 70ecd54254..d97a67fcac 100644 --- a/Sming/Arch/Esp32/Components/sming-arch/component.mk +++ b/Sming/Arch/Esp32/Components/sming-arch/component.mk @@ -18,7 +18,6 @@ COMPONENT_DEPENDS := \ driver \ heap \ fatfs \ - esp_spiffs \ esp32 \ gdbstub \ esptool diff --git a/Sming/Arch/Esp32/app.mk b/Sming/Arch/Esp32/app.mk index a78c572e86..5abbbdc801 100644 --- a/Sming/Arch/Esp32/app.mk +++ b/Sming/Arch/Esp32/app.mk @@ -43,27 +43,6 @@ $(TARGET_OUT): $(COMPONENTS_AR) $(TARGET_BIN): $(TARGET_OUT) $(Q) $(ESPTOOL_CMDLINE) elf2image --min-rev 0 --elf-sha256-offset 0xb0 $(flashimageoptions) -o $@ $< -##@Flashing - -# Partitions -PARTITIONS_CSV ?= $(BUILD_BASE)/partitions.csv -PARTITIONS_BIN = $(FW_BASE)/partitions.bin - -CUSTOM_TARGETS += $(PARTITIONS_CSV) - -$(BUILD_BASE)/partitions.csv: | $(BUILD_BASE) - $(Q) cp $(SDK_PARTITION_PATH)/base.csv $@ - @echo "storage, data, spiffs, $(SPIFF_START_ADDR), $(SPIFF_SIZE)," >> $@ - -$(PARTITIONS_BIN): $(PARTITIONS_CSV) - $(Q) $(ESP32_PYTHON) $(SDK_COMPONENTS_PATH)/partition_table/gen_esp32part.py $< $@ - -.PHONY: partitions -partitions: $(PARTITIONS_BIN) ##Generate partitions table - - -FLASH_PARTITION_CHUNKS := 0x8000=$(PARTITIONS_BIN) - # Application FLASH_APP_CHUNKS := 0x10000=$(TARGET_BIN) @@ -72,13 +51,6 @@ FLASH_APP_CHUNKS := 0x10000=$(TARGET_BIN) flashboot: $(FLASH_BOOT_LOADER) ##Write just the Bootloader $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) -.PHONY: flashconfig - -flashconfig: partitions kill_term ##Write partition config - $(call WriteFlash,$(FLASH_PARTITION_CHUNKS)) - -flashpartition: flashconfig - .PHONY: flashapp flashapp: all kill_term ##Write just the application image $(call WriteFlash,$(FLASH_APP_CHUNKS)) diff --git a/Sming/Arch/Esp8266/Components/esp8266/component.mk b/Sming/Arch/Esp8266/Components/esp8266/component.mk index 46c5d142c8..3d7197df48 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/component.mk +++ b/Sming/Arch/Esp8266/Components/esp8266/component.mk @@ -8,10 +8,11 @@ FLASH_INIT_DATA = $(SDK_BASE)/bin/esp_init_data_default.bin CUSTOM_TARGETS += $(FLASH_INIT_DATA) +# PHY init / System calibration / RF calibration FLASH_INIT_CHUNKS += \ - $(call FlashOffset,0x5000)=$(BLANK_BIN) \ - $(call FlashOffset,0x4000)=$(FLASH_INIT_DATA) \ - $(call FlashOffset,0x2000)=$(BLANK_BIN) + 0x3000=$(FLASH_INIT_DATA) \ + 0x5000=$(BLANK_BIN) \ + 0x6000=$(BLANK_BIN) # => 'Internal' SDK - for SDK Version 3+ as submodule in Sming repository # SDK_BASE just needs to point into our repo as it's overridden with the correct submodule path diff --git a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp index f2adb9e490..3098cac8db 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp +++ b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp @@ -14,6 +14,7 @@ #include #include #include +#include extern void init(); @@ -54,46 +55,9 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void) extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void) { - enum flash_size_map size_map = system_get_flash_size_map(); - uint32 rf_cal_sec = 0; - - switch (int(size_map)) { - case FLASH_SIZE_2M: - rf_cal_sec = 64 - 5; - break; - - case FLASH_SIZE_4M_MAP_256_256: - rf_cal_sec = 128 - 5; - break; - - case FLASH_SIZE_8M_MAP_512_512: - rf_cal_sec = 256 - 5; - break; - - case FLASH_SIZE_16M_MAP_512_512: - case FLASH_SIZE_16M_MAP_1024_1024: - rf_cal_sec = 512 - 5; - break; - - case FLASH_SIZE_32M_MAP_512_512: - case FLASH_SIZE_32M_MAP_1024_1024: - rf_cal_sec = 1024 - 5; - break; - - case 8: // FLASH_SIZE_64M_MAP_1024_1024 - rf_cal_sec = 2048 - 5; - break; - - case 9: // FLASH_SIZE_128M_MAP_1024_1024 - rf_cal_sec = 4096 - 5; - break; - - default: - rf_cal_sec = 0; - break; - } - - return rf_cal_sec; + // RF calibration stored in last sector of sysParam + auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam); + return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1; } #ifdef SDK_INTERNAL @@ -104,25 +68,19 @@ extern "C" uint32 ICACHE_FLASH_ATTR WEAK_ATTR user_rf_cal_sector_set(void) extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void) { - const uint32_t MAX_PROGRAM_SECTORS = 0x100000 / SPI_FLASH_SEC_SIZE; // 1MB addressable + Storage::initialize(); - // WARNING: Sming supports SDK 3.0 with rBoot enabled apps ONLY! - const partition_type_t SYSTEM_PARTITION_RBOOT_CONFIG = static_cast(SYSTEM_PARTITION_CUSTOMER_BEGIN + 0); - const partition_type_t SYSTEM_PARTITION_PROGRAM = static_cast(SYSTEM_PARTITION_CUSTOMER_BEGIN + 1); + auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam); + auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy); - // Partitions offsets and sizes must be in sector multiples, so work in sectors - #define PARTITION_ITEM(_type, _start, _length) \ - {_type, (_start) * SPI_FLASH_SEC_SIZE, (_length) * SPI_FLASH_SEC_SIZE} + // RF calibration stored in last sector of sysParam + auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE; - // Partitions in position order - uint32_t rfCalSector = user_rf_cal_sector_set(); static const partition_item_t partitions[] = { - PARTITION_ITEM(SYSTEM_PARTITION_BOOTLOADER, 0, 1), - PARTITION_ITEM(SYSTEM_PARTITION_RBOOT_CONFIG, 1, 1), - PARTITION_ITEM(SYSTEM_PARTITION_PROGRAM, 2, std::min(MAX_PROGRAM_SECTORS, rfCalSector) - 2), - PARTITION_ITEM(SYSTEM_PARTITION_RF_CAL, rfCalSector, 1), - PARTITION_ITEM(SYSTEM_PARTITION_PHY_DATA, rfCalSector + 1, 1), - PARTITION_ITEM(SYSTEM_PARTITION_SYSTEM_PARAMETER, rfCalSector + 2, 3), + {SYSTEM_PARTITION_BOOTLOADER, 0, SPI_FLASH_SEC_SIZE}, + {SYSTEM_PARTITION_PHY_DATA, phy.address(), phy.size()}, + {SYSTEM_PARTITION_SYSTEM_PARAMETER, sysParam.address(), sysParamSize}, + {SYSTEM_PARTITION_RF_CAL, sysParam.address() + sysParamSize, SPI_FLASH_SEC_SIZE}, }; enum flash_size_map sizeMap = system_get_flash_size_map(); diff --git a/Sming/Arch/Esp8266/app.mk b/Sming/Arch/Esp8266/app.mk index 88f45e491f..149cc37185 100644 --- a/Sming/Arch/Esp8266/app.mk +++ b/Sming/Arch/Esp8266/app.mk @@ -86,7 +86,7 @@ flashapp: all kill_term ##Write just the application image .PHONY: flash flash: all kill_term ##Write the rBoot boot sector, application image and (if enabled) SPIFFS image - $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS) $(FLASH_RBOOT_APP_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) + $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS) $(FLASH_RBOOT_APP_CHUNKS) $(FLASH_PARTITION_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) ifeq ($(ENABLE_GDB), 1) $(GDB_CMDLINE) else diff --git a/Sming/Arch/Host/Components/hostlib/startup.cpp b/Sming/Arch/Host/Components/hostlib/startup.cpp index 330c6c44f7..e9f95c91b2 100644 --- a/Sming/Arch/Host/Components/hostlib/startup.cpp +++ b/Sming/Arch/Host/Components/hostlib/startup.cpp @@ -32,6 +32,7 @@ #include #include #include "include/hostlib/CommandLine.h" +#include #include #include @@ -213,6 +214,8 @@ int main(int argc, char* argv[]) if(config.initonly) { hostmsg("Initialise-only requested"); } else { + Storage::initialize(); + CThread::startup(); hw_timer_init(); diff --git a/Sming/Arch/Host/app.mk b/Sming/Arch/Host/app.mk index 8e26c5a03f..ab11f2bd58 100644 --- a/Sming/Arch/Host/app.mk +++ b/Sming/Arch/Host/app.mk @@ -35,7 +35,7 @@ valgrind: all ##Run the application under valgrind to detect memory issues. Requ RUN_SCRIPT := $(FW_BASE)/run.sh .PHONY: run -run: all $(RUN_SCRIPT) ##Run the application image +run: all flashpart $(RUN_SCRIPT) ##Run the application image $(Q) $(RUN_SCRIPT) $(RUN_SCRIPT):: @@ -45,4 +45,4 @@ $(RUN_SCRIPT):: chmod a+x $@ .PHONY: flash -flash: all flashfs ##Write all images to (virtual) flash +flash: all flashfs flashpart ##Write all images to (virtual) flash diff --git a/Sming/Components/esptool/component.mk b/Sming/Components/esptool/component.mk index c2f8d6d22c..f982afc1c2 100644 --- a/Sming/Components/esptool/component.mk +++ b/Sming/Components/esptool/component.mk @@ -37,7 +37,6 @@ flashimageoptions += -fs $(SPI_SIZE)B endif FLASH_SIZE := $(subst M,*1024K,$(SPI_SIZE)) FLASH_SIZE := $(subst K,*1024,$(FLASH_SIZE)) -FlashOffset = $$(($(FLASH_SIZE)-$1)) BLANK_BIN := $(COMPONENT_PATH)/blank.bin # Default COM port and speed used for flashing diff --git a/Sming/Components/rboot/appcode/rboot-overrides.c b/Sming/Components/rboot/appcode/rboot-overrides.c deleted file mode 100644 index 04132ad0cf..0000000000 --- a/Sming/Components/rboot/appcode/rboot-overrides.c +++ /dev/null @@ -1,43 +0,0 @@ -/**** - * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development. - * Created 2015 by Skurydin Alexey - * http://github.com/SmingHub/Sming - * All files of the Sming Core are provided under the LGPL v3 license. - * - * rboot-overrides.c - * - ****/ - -#include "spiffs_sming.h" -#include - -/* - * rBoot uses different spiffs organization and we need to override this method - * during application compile time in order to make automatic - * mounting with `spiffs_mount()` work as expected. - */ -spiffs_config spiffs_get_storage_config() -{ - spiffs_config cfg = {0}; - u32_t max_allowed_sector, requested_sector; - -#ifdef RBOOT_SPIFFS_0 - cfg.phys_addr = RBOOT_SPIFFS_0; -#elif RBOOT_SPIFFS_1 - cfg.phys_addr = RBOOT_SPIFFS_1; -#else -#error "Define either RBOOT_SPIFFS_0 or RBOOT_SPIFFS_1" -#endif - - cfg.phys_addr &= 0xFFFFF000; // get the start address of the sector - - max_allowed_sector = flashmem_get_sector_of_address(INTERNAL_FLASH_SIZE - 1); - requested_sector = flashmem_get_sector_of_address((cfg.phys_addr + SPIFF_SIZE) - 1); - if(requested_sector > max_allowed_sector) { - debug_w("The requested SPIFFS size is too big."); - requested_sector = max_allowed_sector; - } - // get the max size until the sector end - cfg.phys_size = ((requested_sector + 1) * INTERNAL_FLASH_SECTOR_SIZE) - cfg.phys_addr; - return cfg; -} diff --git a/Sming/Components/rboot/component.mk b/Sming/Components/rboot/component.mk index 7cd47cbdeb..e8fb9993c4 100644 --- a/Sming/Components/rboot/component.mk +++ b/Sming/Components/rboot/component.mk @@ -7,8 +7,8 @@ RBOOT_EMULATION := 1 endif COMPONENT_SUBMODULES := rboot -COMPONENT_INCDIRS := rboot appcode rboot/appcode include -COMPONENT_SRCDIRS := src +COMPONENT_INCDIRS := rboot rboot/appcode include +COMPONENT_SRCDIRS := src src/Arch/$(SMING_ARCH) RBOOT_DIR := $(COMPONENT_PATH) @@ -39,8 +39,8 @@ endif CONFIG_VARS += RBOOT_ROM0_ADDR RBOOT_ROM1_ADDR RBOOT_ROM2_ADDR -# Loation of first ROM (default is sector 2 after rboot and rboot config sector) -RBOOT_ROM0_ADDR ?= 0x002000 +# Location of first ROM +RBOOT_ROM0_ADDR ?= 0x008000 # The parameter below specifies the location of the second ROM. # You need a second slot for any kind of firmware update mechanism. @@ -93,7 +93,9 @@ RBOOT_SPIFFS_1 ?= 0x300000 APP_CFLAGS += -DRBOOT_SPIFFS_0=$(RBOOT_SPIFFS_0) APP_CFLAGS += -DRBOOT_SPIFFS_1=$(RBOOT_SPIFFS_1) +ifneq ($(SPI_SIZE),1M) SPIFF_START_ADDR ?= $(RBOOT_SPIFFS_0) +endif # filenames and options for generating rBoot rom images with esptool2 RBOOT_E2_SECTS ?= .text .text1 .data .rodata @@ -103,7 +105,7 @@ RBOOT_ROM_0_BIN := $(FW_BASE)/$(RBOOT_ROM_0).bin RBOOT_ROM_1_BIN := $(FW_BASE)/$(RBOOT_ROM_1).bin -COMPONENT_APPCODE := appcode rboot/appcode $(if $(RBOOT_EMULATION),host) +COMPONENT_APPCODE := rboot/appcode APP_CFLAGS += -DRBOOT_INTEGRATION # these are exported for use by the rBoot Makefile @@ -133,11 +135,17 @@ ifeq ($(RBOOT_GPIO_SKIP_ENABLED),1) APP_CFLAGS += -DBOOT_GPIO_SKIP_ENABLED endif +COMPONENT_CXXFLAGS += \ + -DRBOOT_ROM0_ADDR=$(RBOOT_ROM0_ADDR) \ + -DRBOOT_ROM1_ADDR=$(RBOOT_ROM1_ADDR) + ifndef RBOOT_EMULATION +export RBOOT_ROM0_ADDR +export RBOOT_ROM1_ADDR RBOOT_BIN := $(FW_BASE)/rboot.bin CUSTOM_TARGETS += $(RBOOT_BIN) $(RBOOT_BIN): - $(Q) $(MAKE) -C $(RBOOT_DIR)/rboot + $(Q) $(MAKE) -C $(RBOOT_DIR)/rboot $(RBOOT_CFLAGS) EXTRA_LDFLAGS := -u Cache_Read_Enable_New diff --git a/Sming/Components/rboot/appcode/rboot-integration.h b/Sming/Components/rboot/include/rboot-integration.h similarity index 100% rename from Sming/Components/rboot/appcode/rboot-integration.h rename to Sming/Components/rboot/include/rboot-integration.h diff --git a/Sming/Components/rboot/host/rboot.cpp b/Sming/Components/rboot/src/Arch/Host/rboot.cpp similarity index 89% rename from Sming/Components/rboot/host/rboot.cpp rename to Sming/Components/rboot/src/Arch/Host/rboot.cpp index 755d942c8d..96c5719d88 100644 --- a/Sming/Components/rboot/host/rboot.cpp +++ b/Sming/Components/rboot/src/Arch/Host/rboot.cpp @@ -40,12 +40,12 @@ void host_init_bootloader() romconf.magic = BOOT_CONFIG_MAGIC; romconf.version = BOOT_CONFIG_VERSION; romconf.count = 2; - romconf.roms[0] = SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1); + romconf.roms[0] = RBOOT_ROM0_ADDR; #ifdef BOOT_ROM1_ADDR - romconf.roms[1] = BOOT_ROM1_ADDR; + romconf.roms[1] = RBOOT_ROM1_ADDR; #else size_t flashsize = flashmem_get_size_bytes(); - romconf.roms[1] = (flashsize / 2) + (SECTOR_SIZE * (BOOT_CONFIG_SECTOR + 1)); + romconf.roms[1] = RBOOT_ROM0_ADDR + (flashsize / 2); #endif bool ok = rboot_set_config(&romconf); hostmsg("Write default config: %s", ok ? "OK" : "FAIL"); diff --git a/Sming/Components/spiffs/README.rst b/Sming/Components/spiffs/README.rst index 7f2792eb57..cefeace69b 100644 --- a/Sming/Components/spiffs/README.rst +++ b/Sming/Components/spiffs/README.rst @@ -1,7 +1,7 @@ SPIFFS for Sming ================ -This Component provides additional functionality to support SPIFFS running on both Esp8266 and Host architectures. +This Component provides SPIFFS filesystem support for all architectures. .. envvar:: DISABLE_SPIFFS diff --git a/Sming/Components/spiffs/spiffs_config.h b/Sming/Components/spiffs/spiffs_config.h index fe7c0b6dd7..1cd0560bcc 100644 --- a/Sming/Components/spiffs/spiffs_config.h +++ b/Sming/Components/spiffs/spiffs_config.h @@ -181,9 +181,7 @@ #endif // Enable this if you want the HAL callbacks to be called with the spiffs struct -#ifndef SPIFFS_HAL_CALLBACK_EXTRA -#define SPIFFS_HAL_CALLBACK_EXTRA 0 -#endif +#define SPIFFS_HAL_CALLBACK_EXTRA 1 // Enable this if you want to add an integer offset to all file handles // (spiffs_file). This is useful if running multiple instances of spiffs on diff --git a/Sming/Components/spiffs/spiffs_sming.cpp b/Sming/Components/spiffs/spiffs_sming.cpp index 0168d93b2b..b138e3517e 100644 --- a/Sming/Components/spiffs/spiffs_sming.cpp +++ b/Sming/Components/spiffs/spiffs_sming.cpp @@ -10,10 +10,13 @@ #include "spiffs_sming.h" #include +#include extern "C" { #include "spiffs/src/spiffs_nucleus.h" } +#define LOG_PAGE_SIZE 256 + spiffs _filesystemStorageHandle; #ifndef SPIFF_FILEDESC_COUNT @@ -26,32 +29,51 @@ uint16_t spiffs_work_buf[LOG_PAGE_SIZE]; spiffs_fd spiffs_fds[SPIFF_FILEDESC_COUNT]; uint32_t spiffs_cache_buf[LOG_PAGE_SIZE + 32]; -s32_t api_spiffs_read(u32_t addr, u32_t size, u8_t* dst) +#define GET_DEVICE() \ + if(fs == nullptr || fs->user_data == nullptr) { \ + debug_e("[SPIFFS] NO DEVICE"); \ + return 0; \ + } \ + auto device = static_cast(fs->user_data); + +s32_t api_spiffs_read(struct spiffs_t* fs, u32_t addr, u32_t size, u8_t* dst) { - return (flashmem_read(dst, addr, size) == size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; + GET_DEVICE(); + return device->read(addr, dst, size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; } -s32_t api_spiffs_write(u32_t addr, u32_t size, u8_t* src) +s32_t api_spiffs_write(struct spiffs_t* fs, u32_t addr, u32_t size, u8_t* src) { //debugf("api_spiffs_write"); - return (flashmem_write(src, addr, size) == size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; + GET_DEVICE(); + return device->write(addr, src, size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; } -s32_t api_spiffs_erase(u32_t addr, u32_t size) +s32_t api_spiffs_erase(struct spiffs_t* fs, u32_t addr, u32_t size) { - debugf("api_spiffs_erase"); - uint32_t sect_first = flashmem_get_sector_of_address(addr); - uint32_t sect_last = sect_first; - while(sect_first <= sect_last) { - if(!flashmem_erase_sector(sect_first++)) { - return SPIFFS_ERR_INTERNAL; - } - } - return SPIFFS_OK; + debugf("api_spiffs_erase(0x%08x, 0x%08x)", addr, size); + GET_DEVICE(); + return device->erase_range(addr, size) ? SPIFFS_OK : SPIFFS_ERR_INTERNAL; } -bool tryMount(const spiffs_config& cfg) +spiffs_config initConfig(Storage::Partition& partition) { + _filesystemStorageHandle.user_data = partition.getDevice(); + return spiffs_config{ + .hal_read_f = api_spiffs_read, + .hal_write_f = api_spiffs_write, + .hal_erase_f = api_spiffs_erase, + .phys_size = partition.size(), + .phys_addr = partition.address(), + .phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE, + .log_block_size = INTERNAL_FLASH_SECTOR_SIZE * 2, + .log_page_size = LOG_PAGE_SIZE, + }; +} + +bool tryMount(Storage::Partition& partition) +{ + auto cfg = initConfig(partition); int res = SPIFFS_mount(&_filesystemStorageHandle, const_cast(&cfg), reinterpret_cast(spiffs_work_buf), reinterpret_cast(spiffs_fds), sizeof(spiffs_fds), spiffs_cache_buf, sizeof(spiffs_cache_buf), nullptr); @@ -60,10 +82,10 @@ bool tryMount(const spiffs_config& cfg) return res >= 0; } -bool spiffs_format_internal(const spiffs_config& cfg) +bool spiffs_format_internal(Storage::Partition& partition) { spiffs_unmount(); - if(tryMount(cfg)) { + if(tryMount(partition)) { spiffs_unmount(); } @@ -71,22 +93,23 @@ bool spiffs_format_internal(const spiffs_config& cfg) return res >= 0; } -bool spiffs_mount_internal(const spiffs_config& cfg) +bool spiffs_mount_internal(Storage::Partition& partition) { + auto cfg = initConfig(partition); debugf("fs.start: size:%u Kb, offset:0x%X\n", cfg.phys_size / 1024U, cfg.phys_addr); // Simple check of the erase count to see if flash looks like it's already been formatted - spiffs_obj_id dat; - flashmem_read(&dat, cfg.phys_addr + cfg.log_page_size - sizeof(spiffs_obj_id), sizeof(spiffs_obj_id)); + spiffs_obj_id dat{UINT16_MAX}; + partition.read(cfg.log_page_size - sizeof(spiffs_obj_id), &dat, sizeof(dat)); //debugf("%X", dat); - bool isFormatted = (dat != spiffs_obj_id(UINT32_MAX)); + bool isFormatted = (dat != UINT16_MAX); if(!isFormatted) { debugf("First init file system"); - spiffs_format_internal(cfg); + spiffs_format_internal(partition); } - if(!tryMount(cfg)) { + if(!tryMount(partition)) { return false; } @@ -102,42 +125,30 @@ bool spiffs_mount_internal(const spiffs_config& cfg) return true; } -bool initConfig(spiffs_config& cfg, uint32_t phys_addr, uint32_t phys_size) +Storage::Partition findDefaultPartition() { - if(phys_addr == 0) { - SYSTEM_ERROR("SPIFFS: Start address invalid"); - return false; + auto it = Storage::findPartition(Storage::Partition::SubType::Data::spiffs); + if(!it) { + debug_e("No SPIFFS partition found"); } - - cfg = spiffs_config{ - .hal_read_f = api_spiffs_read, - .hal_write_f = api_spiffs_write, - .hal_erase_f = api_spiffs_erase, - .phys_size = phys_size, - .phys_addr = phys_addr, - .phys_erase_block = INTERNAL_FLASH_SECTOR_SIZE, - .log_block_size = INTERNAL_FLASH_SECTOR_SIZE * 2, - .log_page_size = LOG_PAGE_SIZE, - }; - - return true; + return *it; } } // namespace bool spiffs_mount() { - spiffs_config cfg = spiffs_get_storage_config(); - return spiffs_mount_manual(cfg.phys_addr, cfg.phys_size); + auto part = findDefaultPartition(); + return part ? spiffs_mount_internal(part) : false; } -bool spiffs_mount_manual(uint32_t phys_addr, uint32_t phys_size) +bool spiffs_mount(Storage::Partition partition) { - spiffs_config cfg; - if(!initConfig(cfg, phys_addr, phys_size)) { + if(!partition.verify(Storage::Partition::SubType::Data::spiffs)) { return false; } - return spiffs_mount_internal(cfg); + + return spiffs_mount_internal(partition); } void spiffs_unmount() @@ -147,16 +158,21 @@ void spiffs_unmount() bool spiffs_format() { - auto cfg = spiffs_get_storage_config(); - return spiffs_format_manual(cfg.phys_addr, cfg.phys_size); + auto part = findDefaultPartition(); + if(!part) { + return false; + } + + spiffs_format_internal(part); + return spiffs_mount_internal(part); } -bool spiffs_format_manual(uint32_t phys_addr, uint32_t phys_size) +bool spiffs_format(Storage::Partition& partition) { - spiffs_config cfg; - if(!initConfig(cfg, phys_addr, phys_size)) { + if(!partition.verify(Storage::Partition::SubType::Data::spiffs)) { return false; } - spiffs_format_internal(cfg); - return spiffs_mount_internal(cfg); + + spiffs_format_internal(partition); + return spiffs_mount_internal(partition); } diff --git a/Sming/Components/spiffs/spiffs_sming.h b/Sming/Components/spiffs/spiffs_sming.h index c8619f9d81..3b969a7d91 100644 --- a/Sming/Components/spiffs/spiffs_sming.h +++ b/Sming/Components/spiffs/spiffs_sming.h @@ -9,14 +9,8 @@ ****/ #pragma once -#if defined(__cplusplus) -extern "C" { -#endif - #include "spiffs.h" -#include - -#define LOG_PAGE_SIZE 256 +#include /** * @brief Mount the SPIFFS volume using default configuration @@ -27,46 +21,31 @@ extern "C" { bool spiffs_mount(); /** - * @brief Mount a SPIFFS volume using custom location and size - * @param phys_addr The flash memory address (offset) for the volume - * @param phys_size The volume size, in bytes - * @retval bool true on success, false on failure - * @note If the given flash memory range appears to be empty then it is - * formatted, erasing any existing content. + * @brief Mount SPIFFS volume from a specific partition */ -bool spiffs_mount_manual(uint32_t phys_addr, uint32_t phys_size); +bool spiffs_mount(Storage::Partition partition); /** - * @brief Unmount a previously mounted volume + * @brief unmount SPIFFS filesystem + * @deprecated use fileFreeFileSystem() instead + * @note this will do nothing if the active filesystem is not SPIFFS */ void spiffs_unmount(); -/** - * @brief Format and mount a SPIFFS volume using default configuration - * @retval bool true on success +/** @brief Format and mount a SPIFFS filesystem + * @deprecated use fileSystemFormat() instead + * @note this will fail if the active filesystem is not SPIFFS */ bool spiffs_format(); /** - * @brief Format and mount a SPIFFS volume using custom location and size - * @param phys_addr The flash memory address (offset) for the volume - * @param phys_size The volume size, in bytes + * @brief Format and mount a SPIFFS volume using given partition + * @param partition * @retval bool true on success */ -bool spiffs_format_manual(uint32_t phys_addr, uint32_t phys_size); - -/** - * @brief Obtain the default SPIFFS configuration information - * @retval spiffs_config - * @note Only `phys_addr` and `phys_size` are used, all other parameters are overridden. - */ -spiffs_config spiffs_get_storage_config(); +bool spiffs_format(Storage::Partition& partition); /** * @brief Global SPIFFS instance used by FileSystem API */ extern spiffs _filesystemStorageHandle; - -#if defined(__cplusplus) -} -#endif diff --git a/Sming/Components/spiffs/spiffy/spiffy.c b/Sming/Components/spiffs/spiffy/spiffy.c index 485e4a87c5..6dab9cedac 100644 --- a/Sming/Components/spiffs/spiffy/spiffy.c +++ b/Sming/Components/spiffs/spiffy/spiffy.c @@ -33,7 +33,7 @@ void hexdump_mem(u8_t *b, u32_t len) { if ((i % 16) != 0) S_DBG("\n"); } -static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) { +static s32_t my_spiffs_read(struct spiffs_t* fs, u32_t addr, u32_t size, u8_t *dst) { int res; @@ -53,7 +53,7 @@ static s32_t my_spiffs_read(u32_t addr, u32_t size, u8_t *dst) { return SPIFFS_OK; } -static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) { +static s32_t my_spiffs_write(struct spiffs_t* fs, u32_t addr, u32_t size, u8_t *src) { int ret = SPIFFS_OK; u8_t *buff = 0; @@ -63,7 +63,7 @@ static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) { printf("Unable to malloc %d bytes.\n", size); ret = SPIFFS_ERR_INTERNAL; } else { - ret = my_spiffs_read(addr, size, buff); + ret = my_spiffs_read(fs, addr, size, buff); if (ret == SPIFFS_OK) { int i; for(i = 0; i < size; i++) buff[i] &= src[i]; @@ -87,7 +87,7 @@ static s32_t my_spiffs_write(u32_t addr, u32_t size, u8_t *src) { return ret; } -static s32_t my_spiffs_erase(u32_t addr, u32_t size) { +static s32_t my_spiffs_erase(struct spiffs_t* fs, u32_t addr, u32_t size) { int i; diff --git a/tests/HostTests/modules/Spiffs.cpp b/tests/HostTests/modules/Spiffs.cpp index f7bdf9b98a..38b3284724 100644 --- a/tests/HostTests/modules/Spiffs.cpp +++ b/tests/HostTests/modules/Spiffs.cpp @@ -32,8 +32,6 @@ class SpiffsTest : public TestGroup DEFINE_FSTR_LOCAL(testFile, "testfile"); DEFINE_FSTR_LOCAL(testContent, "Some test content to write to a file"); - auto cfg = spiffs_get_storage_config(); - // Write to filesystem until sector #0 gets erased unsigned writeCount = 0; uint32_t word0 = 0; @@ -44,7 +42,7 @@ class SpiffsTest : public TestGroup break; } ++writeCount; - if(flashmem_read(&word0, cfg.phys_addr, sizeof(word0)) != sizeof(word0)) { + if(flashmem_read(&word0, _filesystemStorageHandle.cfg.phys_addr, sizeof(word0)) != sizeof(word0)) { debug_e("flashmem_read failed"); TEST_ASSERT(false); break; From a92f1babe1939564496645e56713b16b306ffef7 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Wed, 2 Dec 2020 23:02:37 +0000 Subject: [PATCH 12/36] Add Storage module to HostTests --- tests/HostTests/component.mk | 3 +- tests/HostTests/host-tests.hw | 42 +++++++++++ tests/HostTests/include/modules.h | 1 + tests/HostTests/modules/Spiffs.cpp | 8 +- tests/HostTests/modules/Storage.cpp | 110 ++++++++++++++++++++++++++++ 5 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 tests/HostTests/host-tests.hw create mode 100644 tests/HostTests/modules/Storage.cpp diff --git a/tests/HostTests/component.mk b/tests/HostTests/component.mk index 136f4813d1..284b9f0aab 100644 --- a/tests/HostTests/component.mk +++ b/tests/HostTests/component.mk @@ -1,6 +1,5 @@ -DISABLE_SPIFFS = 1 +HWCONFIG = host-tests DEBUG_VERBOSE_LEVEL = 2 -SPI_SIZE = 4M COMPONENT_INCDIRS := include COMPONENT_SRCDIRS := app modules diff --git a/tests/HostTests/host-tests.hw b/tests/HostTests/host-tests.hw new file mode 100644 index 0000000000..954c4865ff --- /dev/null +++ b/tests/HostTests/host-tests.hw @@ -0,0 +1,42 @@ +{ + "name": "Host Tests profile", + "base_config": "spiffs", + "devices": { + "testDevice": { + "type": "spiram", + "size": "0x40000000" + } + }, + "partitions": { + "spiffs0": { + "size": "0x10000", + "filename": "" + }, + "external1": { + "device": "testDevice", + "address": 0, + "size": "4M", + "type": "data", + "subtype": "spiffs", + "filename": "$(FW_BASE)/test-spiffs.bin", + "build": { + "target": "spiffsgen", + "files": "resource" + } + }, + "external2": { + "device": "testDevice", + "address": "0x600000", + "size": "240K", + "type": "data", + "subtype": 37 + }, + "external3": { + "device": "testDevice", + "address": "0x800000", + "size": "240M", + "type": "data", + "subtype": "nvs" + } + } +} \ No newline at end of file diff --git a/tests/HostTests/include/modules.h b/tests/HostTests/include/modules.h index ba9bfc2520..97583b28b3 100644 --- a/tests/HostTests/include/modules.h +++ b/tests/HostTests/include/modules.h @@ -19,6 +19,7 @@ XX(Url) \ XX(ArduinoJson5) \ XX(ArduinoJson6) \ + XX(Storage) \ XX(Files) \ XX(SpiFlash) \ XX(Spiffs) \ diff --git a/tests/HostTests/modules/Spiffs.cpp b/tests/HostTests/modules/Spiffs.cpp index 38b3284724..67e73475ee 100644 --- a/tests/HostTests/modules/Spiffs.cpp +++ b/tests/HostTests/modules/Spiffs.cpp @@ -1,5 +1,5 @@ #include -#include +#include class SpiffsTest : public TestGroup { @@ -32,6 +32,8 @@ class SpiffsTest : public TestGroup DEFINE_FSTR_LOCAL(testFile, "testfile"); DEFINE_FSTR_LOCAL(testContent, "Some test content to write to a file"); + auto part = *Storage::findPartition(Storage::Partition::SubType::Data::spiffs); + // Write to filesystem until sector #0 gets erased unsigned writeCount = 0; uint32_t word0 = 0; @@ -42,8 +44,8 @@ class SpiffsTest : public TestGroup break; } ++writeCount; - if(flashmem_read(&word0, _filesystemStorageHandle.cfg.phys_addr, sizeof(word0)) != sizeof(word0)) { - debug_e("flashmem_read failed"); + if(!part.read(0, word0)) { + debug_e("part.read() failed"); TEST_ASSERT(false); break; } diff --git a/tests/HostTests/modules/Storage.cpp b/tests/HostTests/modules/Storage.cpp new file mode 100644 index 0000000000..238af5076b --- /dev/null +++ b/tests/HostTests/modules/Storage.cpp @@ -0,0 +1,110 @@ +#include +#include + +class TestDevice : public Storage::Device +{ +public: + String getName() const override + { + return F("testDevice"); + } + + size_t getBlockSize() const override + { + return sizeof(uint32_t); + } + + size_t getSize() const override + { + return 0x40000000; + } + + Type getType() const override + { + return Type::unknown; + } + + bool read(uint32_t address, void* dst, size_t size) override + { + for(unsigned i = 0; i < size; ++i) { + static_cast(dst)[i] = address + i; + } + return true; + } + + bool write(uint32_t address, const void* src, size_t size) override + { + return false; + } + + bool erase_range(uint32_t address, size_t size) override + { + return false; + } +}; + +class PartitionTest : public TestGroup +{ +public: + PartitionTest() : TestGroup(_F("Partition")) + { + } + + void execute() override + { + auto dev = new TestDevice; + Storage::registerDevice(dev); + + listPartitions(); + + delete dev; + } + + void listPartitions() + { + for(auto it = Storage::findPartition(); it; ++it) { + auto part = *it; + + Serial.println(); + Serial.print(_F("Device '")); + Serial.print(part.getDeviceName()); + Serial.print(_F("', partition '")); + Serial.print(part.name()); + Serial.print(_F("', type ")); + Serial.print(int(part.type())); + Serial.print('/'); + Serial.print(int(part.subType())); + Serial.print(" ("); + Serial.print(part.longTypeString()); + Serial.print(_F("), address 0x")); + Serial.print(part.address(), HEX); + Serial.print(_F(", size 0x")); + Serial.print(part.size(), HEX); + Serial.println(); + + testRead(part, 0xE0, 0x20, true); + testRead(part, 10, 20, true); + testRead(part, part.size() - 10, 10, true); + testRead(part, part.size() - 10, 11, false); + } + } + + void testRead(Storage::Partition& part, uint32_t address, size_t size, bool shouldSucceed) + { + auto buf = new uint8_t[size]; + bool success = part.read(address, buf, size); + if(success) { + m_printHex("READ", buf, size, address); + REQUIRE(shouldSucceed == true); + } else { + debug_e("read(0x%08x, %u) failed", address, size); + REQUIRE(shouldSucceed == false); + } + delete[] buf; + } +}; + +void REGISTER_TEST(Storage) +{ + registerGroup(); +} From f19b5b6ef076d4a5370c1cce99b3e79ea716ebb5 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 6 Dec 2020 20:05:08 +0000 Subject: [PATCH 13/36] Fix duplicate code in Basic_rBoot --- samples/Basic_rBoot/app/application.cpp | 39 +++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/samples/Basic_rBoot/app/application.cpp b/samples/Basic_rBoot/app/application.cpp index 945dc409fa..634142f0b1 100644 --- a/samples/Basic_rBoot/app/application.cpp +++ b/samples/Basic_rBoot/app/application.cpp @@ -13,7 +13,7 @@ #define WIFI_PWD "PleaseEnterPass" #endif -RbootHttpUpdater* otaUpdater = 0; +RbootHttpUpdater* otaUpdater; void OtaUpdate_CallBack(RbootHttpUpdater& client, bool result) { @@ -22,10 +22,11 @@ void OtaUpdate_CallBack(RbootHttpUpdater& client, bool result) // success uint8 slot; slot = rboot_get_current_rom(); - if(slot == 0) + if(slot == 0) { slot = 1; - else + } else { slot = 0; + } // set to boot new rom and then reboot Serial.printf("Firmware updated, rebooting to rom %d...\r\n", slot); rboot_set_current_rom(slot); @@ -51,10 +52,11 @@ void OtaUpdate() // select rom slot to flash bootconf = rboot_get_config(); slot = bootconf.current_rom; - if(slot == 0) + if(slot == 0) { slot = 1; - else + } else { slot = 0; + } #ifndef RBOOT_TWO_ROMS // flash rom to position indicated in the rBoot config rom table @@ -90,10 +92,11 @@ void Switch() { uint8 before, after; before = rboot_get_current_rom(); - if(before == 0) + if(before == 0) { after = 1; - else + } else { after = 0; + } Serial.printf("Swapping from rom %d to rom %d.\r\n", before, after); rboot_set_current_rom(after); Serial.println("Restarting...\r\n"); @@ -191,27 +194,27 @@ void init() // mount spiffs int slot = rboot_get_current_rom(); -#if !DISABLE_SPIFFS +#if DISABLE_SPIFFS + debugf("spiffs disabled"); +#else + uint32_t addr; if(slot == 0) { #ifdef RBOOT_SPIFFS_0 - debugf("trying to mount spiffs at 0x%08x, length %d", RBOOT_SPIFFS_0, SPIFF_SIZE); - spiffs_mount_manual(RBOOT_SPIFFS_0, SPIFF_SIZE); + addr = RBOOT_SPIFFS_0; #else - debugf("trying to mount spiffs at 0x%08x, length %d", 0x100000, SPIFF_SIZE); - spiffs_mount_manual(0x100000, SPIFF_SIZE); + addr = 0x100000; #endif } else { #ifdef RBOOT_SPIFFS_1 - debugf("trying to mount spiffs at 0x%08x, length %d", RBOOT_SPIFFS_1, SPIFF_SIZE); - spiffs_mount_manual(RBOOT_SPIFFS_1, SPIFF_SIZE); + addr = RBOOT_SPIFFS_1; #else - debugf("trying to mount spiffs at 0x%08x, length %d", 0x300000, SPIFF_SIZE); - spiffs_mount_manual(0x300000, SPIFF_SIZE); + addr = 0x300000; #endif } -#else - debugf("spiffs disabled"); + debugf("trying to mount spiffs at 0x%08x, length %d", addr, SPIFF_SIZE); + spiffs_mount_manual(addr, SPIFF_SIZE); #endif + WifiAccessPoint.enable(false); Serial.printf("\r\nCurrently running rom %d.\r\n", slot); From d3ef89a938085ca9661284aaab53f05410969215 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 18 Dec 2020 16:42:09 +0000 Subject: [PATCH 14/36] Update rBoot Makefile changes merged upstream make `data` param `const` in `rboot_write_flash()` Correct `offset` from `int` to `uint32_t` --- Sming/Components/rboot/.patches/rboot.patch | 39 +++++--- Sming/Components/rboot/README.rst | 98 ++++++++++++++----- .../rboot/include/Network/RbootHttpUpdater.h | 2 +- Sming/Components/rboot/rboot | 2 +- .../Components/rboot/src/RbootHttpUpdater.cpp | 2 +- .../rboot/src/RbootOutputStream.cpp | 4 +- .../OtaUpgrade/OtaUpgrade/BasicStream.cpp | 2 +- 7 files changed, 104 insertions(+), 45 deletions(-) diff --git a/Sming/Components/rboot/.patches/rboot.patch b/Sming/Components/rboot/.patches/rboot.patch index ee164e45ac..b8cd1ce078 100644 --- a/Sming/Components/rboot/.patches/rboot.patch +++ b/Sming/Components/rboot/.patches/rboot.patch @@ -1,5 +1,5 @@ diff --git a/Makefile b/Makefile -index 0b43474..5176f8f 100644 +index 638a8f7..5176f8f 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,19 @@ endif @@ -22,17 +22,7 @@ index 0b43474..5176f8f 100644 ifneq ($(RBOOT_EXTRA_INCDIR),) CFLAGS += $(addprefix -I,$(RBOOT_EXTRA_INCDIR)) endif -@@ -75,6 +88,10 @@ else ifeq ($(SPI_SIZE), 2Mb) - E2_OPTS += -2048b - else ifeq ($(SPI_SIZE), 4M) - E2_OPTS += -4096 -+else ifeq ($(SPI_SIZE), 8M) -+ E2_OPTS += -8192 -+else ifeq ($(SPI_SIZE), 16M) -+ E2_OPTS += -16384 - endif - ifeq ($(SPI_MODE), qio) - E2_OPTS += -qio + diff --git a/rboot.c b/rboot.c index d622f97..5e254fa 100644 --- a/rboot.c @@ -309,7 +299,7 @@ index d622f97..5e254fa 100644 ets_memcpy((void*)_text_addr, _text_data, _text_len); // return address to load from diff --git a/appcode/rboot-api.c b/appcode/rboot-api.c -index eb4d028..490a763 100644 +index eb4d028..bf276a7 100644 --- a/appcode/rboot-api.c +++ b/appcode/rboot-api.c @@ -9,7 +9,7 @@ @@ -356,6 +346,15 @@ index eb4d028..490a763 100644 return true; } +@@ -103,7 +103,7 @@ bool ICACHE_FLASH_ATTR rboot_write_end(rboot_write_status *status) { + + // function to do the actual writing to flash + // call repeatedly with more data (max len per write is the flash sector size (4k)) +-bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, uint8_t *data, uint16_t len) { ++bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, const uint8_t *data, uint16_t len) { + + bool ret = false; + uint8_t *buffer; @@ -114,7 +114,7 @@ bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, uint8_t *da } @@ -387,3 +386,17 @@ index eb4d028..490a763 100644 return ret; } +diff --git a/appcode/rboot-api.h b/appcode/rboot-api.h +index a98c209..ec4f0e5 100644 +--- a/appcode/rboot-api.h ++++ b/appcode/rboot-api.h +@@ -93,7 +93,7 @@ bool ICACHE_FLASH_ATTR rboot_write_end(rboot_write_status *status); + * of OTA data is received over the network. + * @note Call rboot_write_init before calling this function to get the rboot_write_status structure + */ +-bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, uint8_t *data, uint16_t len); ++bool ICACHE_FLASH_ATTR rboot_write_flash(rboot_write_status *status, const uint8_t *data, uint16_t len); + + #ifdef BOOT_RTC_ENABLED + /** @brief Get rBoot status/control data from RTC data area + diff --git a/Sming/Components/rboot/README.rst b/Sming/Components/rboot/README.rst index df119b1787..50cd239c2a 100644 --- a/Sming/Components/rboot/README.rst +++ b/Sming/Components/rboot/README.rst @@ -7,7 +7,12 @@ Introduction rBoot is a second-stage bootloader that allows booting application images from several pre-configured flash memory addresses, called "slots". Sming supports up to three slots. -Sming uses rBoot exclusively because of its flexibility, reliability and ease of use. +.. note:: + + With Sming 4.3 partitions are used to manage flash storage. + A "slot" refers to a specific application partition, namely ``rom0``, ``rom1`` or ``rom2``. + + The location or size of these partitions is determined by the :ref:`hardware_config`. .. attention:: @@ -19,38 +24,40 @@ Sming uses rBoot exclusively because of its flexibility, reliability and ease of Slot 0 ------ -This is the default slot which is always used. +This is the default slot (``rom0``, the primary application partition) which is always used. .. envvar:: RBOOT_ROM0_ADDR - default: 0x2000. + [read-only] - This is the start address for slot 0. The default is sector 3, immediately after rBoot and its configuration data. + This is the start address for slot 0. - Except for the use case described in `Slot2`_ below, you should not need to set this value. + Except for the use case described in `Slot2`_ below, you should not need to change this. Slot 1 ------ .. envvar:: RBOOT_ROM1_ADDR - default: disabled + [read-only] default: disabled - The start address of slot 1. If you don't need it, leave unconfigured (empty). + The start address of slot 1. If your application includes any kind of Over-the-Air (OTA) firmware update functionality, you will need a second memory slot to store the received update image while the update routines execute from the first slot. +.. note:: + + The ``spiffs-two-roms`` configuration can be used for this purpose. + Upon successful completion of the update, the second slot is activated, such that on next reset rBoot boots into the uploaded application. While now running from slot 1, the next update will be stored to slot 0 again, i.e. the roles of slot 0 and slot 1 are flipped with every update. -For devices with more than 1MB of flash memory, it is advisable to choose an address with the same -offset within its 1MB block as :envvar:`RBOOT_ROM0_ADDR`, e.g. 0x102000 for the default -slot 0 address (0x2000). This way, the same application image can be used for both -slots. See :ref:`single_vs_dual` for further details. +For devices with more than 1MB of flash memory, it is advisable to choose an address +for ``rom1`` with the same offset within its 1MB block as ``rom0``. .. _Slot2: @@ -69,23 +76,61 @@ To enable slot 2, set these values: .. envvar:: RBOOT_ROM2_ADDR - Address for slot 2 + [read-only] + + Address for slot 2. You must create a custom :ref:`hardware_config` for your project + with a definition for ``rom2``. + + .. code-block:: json + + { + ... + "partitions": { + "rom2": { + "address": "0x100000", + "size": "512K", + "type": "app", + "subtype": "ota_1" + } + } + } + +.. note:: + + At present, this will only configure rBoot. + Sming will not create an application image for slot 2. + + You can, however, use a second Sming project to build a recovery application image as follows: + + - Create a new Sming project for your recovery application. This will be a simple + single-slot project. Create a new :ref:`hardware_config` and configure the + ``rom0`` start address and size to the same as the ``rom2`` partition of the main project. + + option (a) + + - Build and flash the recovery project as usual by typing ``make flash``. This will + install the recovery ROM (into slot 2 of the main project) and a temporary + bootloader, which will be overwritten in the next step. + + - Go back to your main project. Build and flash it with ``make flash``. This will + install the main application (into slot 0) and the final bootloader. You are + now all set for booting into the recovery image if the need arises. + + option (b) + + - Run a normal ``make`` for your recovery project + + - Locate the firmware image file, typically ``out/Esp8266/release/firmware/rom0.bin`` + (adjust accordingly if using a debug build). + Copy this image file as ``rom2.bin`` into your main project directory. -Note that this will only configure rBoot. -Sming will not create an application image for slot 2. -You can, however, use a second Sming project to build a recovery application image as follows: + - Add an additional property to the ``rom2`` partition entry in your main project: -1. Create a new Sming project for your recovery application. This will be a simple - single-slot project. Set :envvar:`RBOOT_ROM0_ADDR` of the recovery project to the value - of :envvar:`RBOOT_ROM2_ADDR` of the main project. + .. code-block:: json -2. Build and flash the recovery project as usual by typing ``make flash``. This will - install the recovery ROM (into slot 2 of the main project) and a temporary - bootloader, which will be overwritten in the next step. + "filename": "rom2.bin" -3. Go back to your main project. Build and flash it with ``make flash``. This will - install the main application (into slot 0) and the final bootloader. You are - now all set for booting into the recovery image if the need arises. + When you run ``make flash`` in this will get written along with the other partitions. Automatically derived settings ------------------------------ @@ -122,8 +167,9 @@ the same address offsets within their 1MB blocks, i.e. ``(RBOOT_ROM0_ADDR & 0xFF Consequently, for such configurations, the Sming build system generates only one ROM image. In all other cases, two distinct application images must be linked with different addresses -for the 'irom0_0_seg' memory region. The Sming build system takes care of all the details, -including linker script generation. +for the 'irom0_0_seg' memory region. +You should use the ``two-rom-mode`` :ref:`hardware_config` for this. +The Sming build system will handle everything else, including linker script generation. .. envvar:: RBOOT_TWO_ROMS diff --git a/Sming/Components/rboot/include/Network/RbootHttpUpdater.h b/Sming/Components/rboot/include/Network/RbootHttpUpdater.h index 3a6ac65c46..c7ed16a43c 100644 --- a/Sming/Components/rboot/include/Network/RbootHttpUpdater.h +++ b/Sming/Components/rboot/include/Network/RbootHttpUpdater.h @@ -39,7 +39,7 @@ class RbootHttpUpdater : protected HttpClient cleanup(); } - bool addItem(int offset, const String& firmwareFileUrl, size_t maxSize = 0); + bool addItem(uint32_t offset, const String& firmwareFileUrl, size_t maxSize = 0); bool addItem(const String& firmwareFileUrl, RbootOutputStream* stream = nullptr); void start(); diff --git a/Sming/Components/rboot/rboot b/Sming/Components/rboot/rboot index 24f6021da8..614f33685d 160000 --- a/Sming/Components/rboot/rboot +++ b/Sming/Components/rboot/rboot @@ -1 +1 @@ -Subproject commit 24f6021da82b303262a939c955f6a984bce6208f +Subproject commit 614f33685d0dd990fc4202f2409b0d2365eeaef3 diff --git a/Sming/Components/rboot/src/RbootHttpUpdater.cpp b/Sming/Components/rboot/src/RbootHttpUpdater.cpp index 077d9697da..07d28bf31d 100644 --- a/Sming/Components/rboot/src/RbootHttpUpdater.cpp +++ b/Sming/Components/rboot/src/RbootHttpUpdater.cpp @@ -15,7 +15,7 @@ #include -bool RbootHttpUpdater::addItem(int offset, const String& firmwareFileUrl, size_t maxSize) +bool RbootHttpUpdater::addItem(uint32_t offset, const String& firmwareFileUrl, size_t maxSize) { RbootHttpUpdaterItem add; add.targetOffset = offset; diff --git a/Sming/Components/rboot/src/RbootOutputStream.cpp b/Sming/Components/rboot/src/RbootOutputStream.cpp index 5fea78dff2..e7b71e4eb5 100644 --- a/Sming/Components/rboot/src/RbootOutputStream.cpp +++ b/Sming/Components/rboot/src/RbootOutputStream.cpp @@ -33,12 +33,12 @@ size_t RbootOutputStream::write(const uint8_t* data, size_t size) initialized = true; } - if(maxLength && (written + size > maxLength)) { + if(maxLength != 0 && (written + size > maxLength)) { debug_e("The ROM size is bigger than the maximum allowed"); return 0; } - if(!rboot_write_flash(&rBootWriteStatus, (uint8_t*)data, size)) { + if(!rboot_write_flash(&rBootWriteStatus, data, size)) { debug_e("rboot_write_flash: Failed. Size: %d", size); return 0; } diff --git a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp index ca549eac12..c1ac6be9e4 100644 --- a/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp +++ b/Sming/Libraries/OtaUpgrade/OtaUpgrade/BasicStream.cpp @@ -184,7 +184,7 @@ size_t BasicStream::write(const uint8_t* data, size_t size) break; case State::WriteRom: { - bool ok = rboot_write_flash(&rbootWriteStatus, const_cast(data), std::min(remainingBytes, size)); + bool ok = rboot_write_flash(&rbootWriteStatus, data, std::min(remainingBytes, size)); if(ok) { if(consume(data, size)) { ok = slot.updated = rboot_write_end(&rbootWriteStatus); From a2e1a4543c5b160dcba1692abe34b6117dbad7b8 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Mon, 7 Dec 2020 09:19:58 +0000 Subject: [PATCH 15/36] Update Basic_rBoot --- docs/source/information/rboot-ota.rst | 89 ++++++------------------- samples/Basic_rBoot/README.rst | 14 ++-- samples/Basic_rBoot/app/application.cpp | 73 +++++++++----------- samples/Basic_rBoot/basic_rboot.hw | 21 ++++++ samples/Basic_rBoot/component.mk | 32 ++------- 5 files changed, 90 insertions(+), 139 deletions(-) create mode 100644 samples/Basic_rBoot/basic_rboot.hw diff --git a/docs/source/information/rboot-ota.rst b/docs/source/information/rboot-ota.rst index a2953df1f8..5f22026f90 100644 --- a/docs/source/information/rboot-ota.rst +++ b/docs/source/information/rboot-ota.rst @@ -44,13 +44,16 @@ Configuring for two ROM mode ============================ If you have a 1MB flash, you will need to have two 512KB ROM slots, both -in the same 1MB block of flash. Set the following options in your project's -``component.mk`` file: +in the same 1MB block of flash. You can accommodate this by setting the +appropriate hardware configuration in your project's component.mk file:: .. code-block:: make - RBOOT_ROM1_ADDR = 0x80000 - SPI_SIZE = 1M + HWCONFIG = two-rom-mode + +See ``Sming/Arch/Esp8266/two-rom-mode.hw`` for details. +You can copy this and customise it in your project. + SPIFFS ====== @@ -60,9 +63,10 @@ To use SPIFFS think about where you want your SPIFFS to sit on the flash. If you have a 4MB flash the default position is for the first ROM to be placed in the first 1MB block and the second ROM to be placed in the third 1MB block of flash. This leaves a whole 1MB spare after each -ROM in which you can put your SPIFFS. +ROM in which you can put your SPIFFS. This is the behaviour when you +set ``HWCONFIG = spiffs`` in your project's component.mk file. -If you have to a smaller flash the SPIFFS will have to share the 1MB block with the ROM. +If you have a smaller flash the SPIFFS will have to share the 1MB block with the ROM. For example, the first part of each 1MB block may contain the ROM, and the second part the SPIFFS (but does *not* have to be split equally in half). So for the 4MB example you could put the SPIFFS for your first ROM at flash address at 0x100000 @@ -74,9 +78,14 @@ To mount your SPIFFS at boot time add the following code to init: .. code-block:: c++ int slot = rboot_get_current_rom(); - uint32_t address = (slot == 0) ? RBOOT_SPIFFS_0 : RBOOT_SPIFFS_1; - //debugf("trying to mount SPIFFS at %x, length %d", address, SPIFF_SIZE); - spiffs_mount_manual(address, SPIFF_SIZE); + // Find the n'th SPIFFS partition + auto part = PartitionTable().find(Partition::SubType::Data::spiffs, slot); + if(part) { + //debugf("trying to mount SPIFFS at %x, length %d", part.address(), part.size()); + spiffs_mount(part); + } else { + debug_e("SPIFFS partition missing for slot #%u", slot); + } Over-the-air (OTA) updates ========================== @@ -98,64 +107,6 @@ products. A more lightweight solution is provided by :cpp:class:`RbootOutputStream`, which is just a thin wrapper around rBoot's flash API, in combination with :cpp:class:`RbootHttpUpdater`, -which pulls individual ROM image from an HTTP server. Add the following code: - -.. code-block:: c++ - - RbootHttpUpdater* otaUpdater = nullptr; - - void OtaUpdate_CallBack(RbootHttpUpdater& client, bool result) - { - if (result) { - // success - switch slot - uint8_t slot = rboot_get_current_rom(); - if (slot == 0) { - slot = 1; - } else { - slot = 0; - } - // set to boot new ROM and then reboot - Serial.printf("Firmware updated, rebooting to ROM %d...\r\n", slot); - rboot_set_current_rom(slot); - System.restart(); - } else { - // fail - Serial.println("Firmware update failed!"); - } - } - - void OtaUpdate() - { - // need a clean object, otherwise if run before and failed will not run again - delete otaUpdater; - otaUpdater = new RbootHttpUpdater(); - - // select ROM slot to flash - rboot_config bootconf = rboot_get_config(); - uint8_t slot = bootconf.current_rom; - if (slot == 0) { - slot = 1; - } else { - slot = 0; - } - - #ifndef RBOOT_TWO_ROMS - // flash ROM to position indicated in the rBoot config ROM table - otaUpdater->addItem(bootconf.roms[slot], ROM_0_URL); - #else - // flash appropriate ROM - otaUpdater->addItem(bootconf.roms[slot], (slot == 0) ? ROM_0_URL : ROM_1_URL); - #endif - - // use user supplied values (defaults for 4MB flash in makefile) - otaUpdater->addItem((slot == 0) ? RBOOT_SPIFFS_0 : RBOOT_SPIFFS_1, SPIFFS_URL); - - // set a callback - otaUpdater->setCallback(OtaUpdate_CallBack); - - // start update - otaUpdater->start(); - } +which pulls individual ROM image from an HTTP server. -You will need to define ``ROM_0_URL``, ``ROM_1_URL`` and ``SPIFFS_URL`` -with http urls for the files to download. +For details, refer to the `OtaUpdate()` function in the :sample:`Basic_rBoot` sample. diff --git a/samples/Basic_rBoot/README.rst b/samples/Basic_rBoot/README.rst index 68307498a1..8b1a836953 100644 --- a/samples/Basic_rBoot/README.rst +++ b/samples/Basic_rBoot/README.rst @@ -44,12 +44,20 @@ Technical Notes ``spiffs_mount_manual(address, length)`` must be called from init. +.. note:: + + This method is now deprecated. Please configure partitions appropriately, + use PartitionTable methods to locate the desired partition, then mount it:: + + auto part = PartitionTable().find('spiffs0'); + spiffs_mount(part); + + See :ref:`hardware-config` for further details. + Important compiler flags used: - BOOT_BIG_FLASH - when using big flash mode, ensures flash mapping code is built in to the rom. - RBOOT_INTEGRATION - ensures Sming specific options are pulled in to the rBoot source at compile time. -- SPIFF_SIZE=value - passed through to code for mounting the filesystem. - Also used in the Makefile to create the SPIFFS. Flash layout considerations --------------------------- @@ -62,8 +70,6 @@ See the rBoot readme for further details. - If using a very small flash (e.g. 512k) there may be no room for a spiffs fileystem, disable it with *DISABLE_SPIFFS = 1* -- If you are using spiffs set *RBOOT_SPIFFS_0* & *RBOOT_SPIFFS_1* to - indicate where the filesystems are located on the flash. - After building copy all the rom*.bin files to the root of your web server. diff --git a/samples/Basic_rBoot/app/application.cpp b/samples/Basic_rBoot/app/application.cpp index 634142f0b1..19d0a683c9 100644 --- a/samples/Basic_rBoot/app/application.cpp +++ b/samples/Basic_rBoot/app/application.cpp @@ -1,5 +1,4 @@ #include -#include #include // download urls, set appropriately @@ -14,8 +13,20 @@ #endif RbootHttpUpdater* otaUpdater; +Storage::Partition spiffsPartition; -void OtaUpdate_CallBack(RbootHttpUpdater& client, bool result) +Storage::Partition findSpiffsPartition(uint8_t slot) +{ + String name = F("spiffs"); + name += slot; + auto part = Storage::findPartition(name); + if(!part) { + debug_w("Partition '%s' not found", name.c_str()); + } + return part; +} + +void otaUpdateCallBack(RbootHttpUpdater& client, bool result) { Serial.println("In callback..."); if(result == true) { @@ -45,8 +56,9 @@ void OtaUpdate() Serial.println("Updating..."); // need a clean object, otherwise if run before and failed will not run again - if(otaUpdater) + if(otaUpdater) { delete otaUpdater; + } otaUpdater = new RbootHttpUpdater(); // select rom slot to flash @@ -62,27 +74,20 @@ void OtaUpdate() // flash rom to position indicated in the rBoot config rom table otaUpdater->addItem(bootconf.roms[slot], ROM_0_URL); #else - // flash appropriate rom - if(slot == 0) { - otaUpdater->addItem(bootconf.roms[slot], ROM_0_URL); - } else { - otaUpdater->addItem(bootconf.roms[slot], ROM_1_URL); - } + // flash appropriate ROM + otaUpdater->addItem(bootconf.roms[slot], (slot == 0) ? ROM_0_URL : ROM_1_URL); #endif -#if !DISABLE_SPIFFS - // use user supplied values (defaults for 4mb flash in makefile) - if(slot == 0) { - otaUpdater->addItem(RBOOT_SPIFFS_0, SPIFFS_URL); - } else { - otaUpdater->addItem(RBOOT_SPIFFS_1, SPIFFS_URL); + auto part = findSpiffsPartition(slot); + if(part) { + // use user supplied values (defaults for 4mb flash in hardware config) + otaUpdater->addItem(part.address(), SPIFFS_URL, part.size()); } -#endif // request switch and reboot on success //otaUpdater->switchToRom(slot); // and/or set a callback (called on failure or success without switching requested) - otaUpdater->setCallback(OtaUpdate_CallBack); + otaUpdater->setCallback(otaUpdateCallBack); // start update otaUpdater->start(); @@ -176,10 +181,10 @@ void serialCallBack(Stream& stream, char arrivedChar, unsigned short availableCh Serial.println(" switch - switch to the other rom and reboot"); Serial.println(" ota - perform ota update, switch rom and reboot"); Serial.println(" info - show esp8266 info"); -#if !DISABLE_SPIFFS - Serial.println(" ls - list files in spiffs"); - Serial.println(" cat - show first file in spiffs"); -#endif + if(spiffsPartition) { + Serial.println(" ls - list files in spiffs"); + Serial.println(" cat - show first file in spiffs"); + } Serial.println(); } else { Serial.println("unknown command"); @@ -193,27 +198,13 @@ void init() Serial.systemDebugOutput(true); // Debug output to serial // mount spiffs - int slot = rboot_get_current_rom(); -#if DISABLE_SPIFFS - debugf("spiffs disabled"); -#else - uint32_t addr; - if(slot == 0) { -#ifdef RBOOT_SPIFFS_0 - addr = RBOOT_SPIFFS_0; -#else - addr = 0x100000; -#endif - } else { -#ifdef RBOOT_SPIFFS_1 - addr = RBOOT_SPIFFS_1; -#else - addr = 0x300000; -#endif + auto slot = rboot_get_current_rom(); + spiffsPartition = findSpiffsPartition(slot); + if(spiffsPartition) { + debugf("trying to mount '%s' at 0x%08x, length %d", spiffsPartition.name().c_str(), spiffsPartition.address(), + spiffsPartition.size()); + spiffs_mount(spiffsPartition); } - debugf("trying to mount spiffs at 0x%08x, length %d", addr, SPIFF_SIZE); - spiffs_mount_manual(addr, SPIFF_SIZE); -#endif WifiAccessPoint.enable(false); diff --git a/samples/Basic_rBoot/basic_rboot.hw b/samples/Basic_rBoot/basic_rboot.hw new file mode 100644 index 0000000000..3f3d1161e7 --- /dev/null +++ b/samples/Basic_rBoot/basic_rboot.hw @@ -0,0 +1,21 @@ +{ + "name": "Two ROM slots, two SPIFFS", + "base_config": "spiffs", + "partitions": { + "rom1": { + "address": "0x108000", + "size": "992K", + "type": "app", + "subtype": "ota_0", + "flags": "", + "filename": "$(RBOOT_ROM_1_BIN)" + }, + "spiffs1": { + "address": "0x300000", + "size": "512K", + "type": "data", + "subtype": "spiffs", + "flags": "" + } + } +} diff --git a/samples/Basic_rBoot/component.mk b/samples/Basic_rBoot/component.mk index 1574232c70..d77a3c8c2f 100644 --- a/samples/Basic_rBoot/component.mk +++ b/samples/Basic_rBoot/component.mk @@ -1,31 +1,13 @@ #### overridable rBoot options #### ## use rboot build mode -RBOOT_ENABLED ?= 1 - -## size of the flash chip -SPI_SIZE ?= 4M - -## address of ROM slots 0 & 1 -#RBOOT_ROM0_ADDR ?= 0x002000 -RBOOT_ROM1_ADDR ?= 0x102000 - -## output file for first rom (.bin will be appended) -#RBOOT_ROM_0 ?= rom0 - -## these next options only needed when using two rom mode -#RBOOT_ROM_1 ?= rom1 - -## size of the spiffs to create -SPIFF_SIZE ?= 65536 - -## option to completely disable spiffs -#DISABLE_SPIFFS ?= 1 - -## flash offsets for spiffs, set if using two rom mode or not on a 4mb flash -## (spiffs location defaults to the mb after the rom slot on 4mb flash) -#RBOOT_SPIFFS_0 ?= 0x100000 -#RBOOT_SPIFFS_1 ?= 0x300000 +RBOOT_ENABLED := 1 +## Use standard hardware config with two ROM slots and two SPIFFS partitions +ifeq ($(SMING_ARCH),Esp8266) +HWCONFIG := basic_rboot +else +HWCONFIG := spiffs # Emulate UART 0 ENABLE_HOST_UARTID := 0 +endif From d1fdcc73e6465b9b6d0789d5f640eebbbe03c9dc Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 6 Dec 2020 09:40:13 +0000 Subject: [PATCH 16/36] Handle all flashing via partitions Generate hwconfig.mk for makefile use Set rBoot ROM addresses from partition table Add erasepart target Add vflash utility for more flexible Host support Add spiffsgen target and build via partition table Add readmap, readpart targets Add basic python vflash utility for Host --- Sming/Arch/Esp32/app.mk | 21 --- .../Esp8266/Components/esp8266/component.mk | 6 - Sming/Arch/Esp8266/app.mk | 17 --- Sming/Arch/Host/Components/vflash/README.rst | 2 +- .../Arch/Host/Components/vflash/component.mk | 32 +++-- Sming/Arch/Host/Components/vflash/vflash.py | 123 ++++++++++++++++++ Sming/Arch/Host/app.mk | 5 +- Sming/Components/Storage/component.mk | 109 +++++++++++++--- Sming/Components/esptool/README.rst | 10 +- Sming/Components/esptool/component.mk | 58 +++------ Sming/Components/rboot/component.mk | 33 ++--- Sming/Components/spiffs/component.mk | 73 ++--------- Sming/component-wrapper.mk | 1 + Sming/component.mk | 19 ++- Sming/project.mk | 2 + 15 files changed, 306 insertions(+), 205 deletions(-) create mode 100644 Sming/Arch/Host/Components/vflash/vflash.py diff --git a/Sming/Arch/Esp32/app.mk b/Sming/Arch/Esp32/app.mk index 5abbbdc801..4c9fca76bd 100644 --- a/Sming/Arch/Esp32/app.mk +++ b/Sming/Arch/Esp32/app.mk @@ -42,24 +42,3 @@ $(TARGET_OUT): $(COMPONENTS_AR) $(TARGET_BIN): $(TARGET_OUT) $(Q) $(ESPTOOL_CMDLINE) elf2image --min-rev 0 --elf-sha256-offset 0xb0 $(flashimageoptions) -o $@ $< - -# Application - -FLASH_APP_CHUNKS := 0x10000=$(TARGET_BIN) - -.PHONY: flashboot -flashboot: $(FLASH_BOOT_LOADER) ##Write just the Bootloader - $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) - -.PHONY: flashapp -flashapp: all kill_term ##Write just the application image - $(call WriteFlash,$(FLASH_APP_CHUNKS)) - -.PHONY: flash -flash: all partitions kill_term ##Write the boot loader, application image, partition table and (if enabled) SPIFFS image - $(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_APP_CHUNKS) $(FLASH_PARTITION_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) -ifeq ($(ENABLE_GDB), 1) - $(GDB_CMDLINE) -else - $(TERMINAL) -endif diff --git a/Sming/Arch/Esp8266/Components/esp8266/component.mk b/Sming/Arch/Esp8266/Components/esp8266/component.mk index 3d7197df48..789d7aa24f 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/component.mk +++ b/Sming/Arch/Esp8266/Components/esp8266/component.mk @@ -8,12 +8,6 @@ FLASH_INIT_DATA = $(SDK_BASE)/bin/esp_init_data_default.bin CUSTOM_TARGETS += $(FLASH_INIT_DATA) -# PHY init / System calibration / RF calibration -FLASH_INIT_CHUNKS += \ - 0x3000=$(FLASH_INIT_DATA) \ - 0x5000=$(BLANK_BIN) \ - 0x6000=$(BLANK_BIN) - # => 'Internal' SDK - for SDK Version 3+ as submodule in Sming repository # SDK_BASE just needs to point into our repo as it's overridden with the correct submodule path # This provides backward-compatiblity, so $(SMING)/third-party/ESP8266_NONOS_SDK) still works diff --git a/Sming/Arch/Esp8266/app.mk b/Sming/Arch/Esp8266/app.mk index 149cc37185..2521c1df13 100644 --- a/Sming/Arch/Esp8266/app.mk +++ b/Sming/Arch/Esp8266/app.mk @@ -71,24 +71,7 @@ $(TARGET_OUT_1): $(COMPONENTS_AR) $(LIBMAIN_DST) ##@Flashing -.PHONY: flashboot -flashboot: $(RBOOT_BIN) ##Write just the rBoot boot sector - $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS)) - .PHONY: flashconfig flashconfig: kill_term ##Erase the rBoot config sector $(info Erasing rBoot config sector) $(call WriteFlash,$(FLASH_RBOOT_ERASE_CONFIG_CHUNKS)) - -.PHONY: flashapp -flashapp: all kill_term ##Write just the application image - $(call WriteFlash,$(FLASH_RBOOT_APP_CHUNKS)) - -.PHONY: flash -flash: all kill_term ##Write the rBoot boot sector, application image and (if enabled) SPIFFS image - $(call WriteFlash,$(FLASH_RBOOT_BOOT_CHUNKS) $(FLASH_RBOOT_APP_CHUNKS) $(FLASH_PARTITION_CHUNKS) $(FLASH_SPIFFS_CHUNKS)) -ifeq ($(ENABLE_GDB), 1) - $(GDB_CMDLINE) -else - $(TERMINAL) -endif diff --git a/Sming/Arch/Host/Components/vflash/README.rst b/Sming/Arch/Host/Components/vflash/README.rst index f94b6416f1..862312c3af 100644 --- a/Sming/Arch/Host/Components/vflash/README.rst +++ b/Sming/Arch/Host/Components/vflash/README.rst @@ -19,7 +19,7 @@ The following options are interpreted and used to provide command-line parameter This defaults to a combination of the above variables, but you can override if necessary. -The size of the flash memory is set via :envvar:`SPI_SIZE`. +The size of the flash memory is set via :ref:`hardware_config`. See :component-esp8266:`esptool` for details and other applicable variables. diff --git a/Sming/Arch/Host/Components/vflash/component.mk b/Sming/Arch/Host/Components/vflash/component.mk index 10dc7b47cc..d558aaeda4 100644 --- a/Sming/Arch/Host/Components/vflash/component.mk +++ b/Sming/Arch/Host/Components/vflash/component.mk @@ -15,24 +15,30 @@ CACHE_VARS += HOST_FLASH_OPTIONS HOST_FLASH_OPTIONS ?= --flashfile=$(FLASH_BIN) --flashsize=$(SPI_SIZE) CLI_TARGET_OPTIONS += $(HOST_FLASH_OPTIONS) -# Write data to flash -# $1 -> Start offset -# $2 -> File containing data to write -define WriteFlashChunk - $(info WriteFlash $1 -> $2) - $(Q) if [ ! -f $(FLASH_BIN) ]; then \ - $(EraseFlash); \ - fi - $(Q) $(DD) if=$2 of=$(FLASH_BIN) obs=1 seek=$$(($1)) conv=notrunc -endef +# Virtual flasher tool +VFLASH := $(PYTHON) $(COMPONENT_PATH)/vflash.py $(FLASH_BIN) $(STORAGE_DEVICE_spiFlash_SIZE_BYTES) # Write one or more chunks to flash # $1 -> List of `Offset=File` chunks define WriteFlash - $(foreach c,$1,$(call WriteFlashChunk,$(word 1,$(subst =, ,$c)),$(word 2,$(subst =, ,$c)))) + $(if $1,$(Q) $(VFLASH) write-chunks $1) +endef + +# Read flash memory into file +# $1 -> `Offset,Size` chunk +# $2 -> Output filename +define ReadFlash + $(info ReadFlash $1,$2) + $(Q) $(VFLASH) read-chunks $1=$2 +endef + +# Erase a region of Flash +# $1 -> List of `Offset,Size` chunks +define EraseFlashRegion + $(Q) $(VFLASH) fill-regions $1 endef # Reset/create flash backing file define EraseFlash - $(DD) if=/dev/zero ibs=1 count=$(SPI_SIZE) | tr "\000" "\377" > $(FLASH_BIN) -endef + $(Q) $(VFLASH) erase +endef diff --git a/Sming/Arch/Host/Components/vflash/vflash.py b/Sming/Arch/Host/Components/vflash/vflash.py new file mode 100644 index 0000000000..cf91295cf5 --- /dev/null +++ b/Sming/Arch/Host/Components/vflash/vflash.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# +# Virtual flash tool to support operations on flash backing file for Sming Host. +# +# Copyright 2021 mikee47 +# +# This file is part of the Sming Framework Project +# +# This library is free software: you can redistribute it and/or modify it under the terms of the +# GNU General Public License as published by the Free Software Foundation, version 3 or later. +# +# This library is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +# without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU General Public License for more details. +# + +import os, argparse + +# Interpret argument such as 0x1234 +def auto_int(x): + return int(x, 0) + +def fill_region(offset, length, value): + print(" fill_region(0x%08x, 0x%04x, 0x%02x)" % (offset, length, value)) + flashImage.seek(offset, os.SEEK_SET) + tmp = bytearray([value] * length) + flashImage.write(tmp) + +def erase(args): + fill_region(0, args.imagesize, 0xff) + +def erase_region(args): + fill_region(args.offset, args.size, 0xff) + +# Offset,size +def erase_regions(args): + for s in args.chunks: + offset, size = s.split(',') + fill_region(int(offset, 0), int(size, 0), 0xff) + +# Offset,size=value +def fill_regions(args): + for s in args.chunks: + offset, tmp = s.split(',') + x = tmp.split('=') + if len(x) == 1: + size = x[0] + value = '0xff' + else: + size, value = x + fill_region(int(offset, 0), int(size, 0), int(value,0)) + +def write_chunk(offset, filename): + print(" write_chunk(0x%08x, '%s')" % (offset, filename)) + flashImage.seek(offset, os.SEEK_SET) + data = open(filename, "rb").read() + len = flashImage.write(data) + print(" - wrote 0x%04x bytes" % len) + +# Offset=filename +def write_chunks(args): + for s in args.chunks: + offset, file = s.split('=') + write_chunk(int(offset, 0), file) + + +def read_chunk(offset, size, filename): + print(" read_chunk(0x%08x, 0x%04x, '%s')" % (offset, size, filename)) + flashImage.seek(offset, os.SEEK_SET) + data = flashImage.read(size) + open(filename, "wb").write(data) + +# Offset,size=filename +def read_chunks(args): + for s in args.chunks: + offset, x = s.split(',') + size, filename = x.split('=') + read_chunk(int(offset, 0), int(size, 0), filename) + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description='Virtual flash tool') + parser.add_argument('filename', help='Flash image filename') + parser.add_argument('imagesize', type=auto_int, help='Flash image size') + parser.set_defaults(func=None) + subparsers = parser.add_subparsers() + + fill_regions_parser = subparsers.add_parser('fill-regions', help='Fill region(s) of flash') + fill_regions_parser.add_argument('chunks', nargs='+', help='List of "offset,size[=value]" chunks, default value is 0xff') + fill_regions_parser.set_defaults(func=fill_regions) + + erase_parser = subparsers.add_parser('erase', help='Erase entire flash') + erase_parser.set_defaults(func=erase) + + write_chunks_parser = subparsers.add_parser('write-chunks', help='Write chunks to flash') + write_chunks_parser.add_argument('chunks', nargs='+', help='List of "offset=file" chunks') + write_chunks_parser.set_defaults(func=write_chunks) + + read_chunks_parser = subparsers.add_parser('read-chunks', help='Read chunks from flash') + read_chunks_parser.add_argument('chunks', nargs='+', help='List of "offset,size=file" chunks') + read_chunks_parser.set_defaults(func=read_chunks) + + args = parser.parse_args() + + flashImage = open(args.filename, "rb+" if os.path.isfile(args.filename) else "wb+") + + print("vflash('%s')" % args.filename) + + # Pad flash image file if it's not big enough + flashImage.seek(0, os.SEEK_END) + imgsize = flashImage.tell() + if imgsize < args.imagesize: + fill_region(imgsize, args.imagesize - imgsize, 0xff) + + # Invoke any user-provided functions (optional) + fn = args.func + if fn is None: + parser.print_usage() + else: + fn(args) + + flashImage.close() diff --git a/Sming/Arch/Host/app.mk b/Sming/Arch/Host/app.mk index ab11f2bd58..9e4f858af4 100644 --- a/Sming/Arch/Host/app.mk +++ b/Sming/Arch/Host/app.mk @@ -35,7 +35,7 @@ valgrind: all ##Run the application under valgrind to detect memory issues. Requ RUN_SCRIPT := $(FW_BASE)/run.sh .PHONY: run -run: all flashpart $(RUN_SCRIPT) ##Run the application image +run: all $(RUN_SCRIPT) ##Run the application image $(Q) $(RUN_SCRIPT) $(RUN_SCRIPT):: @@ -43,6 +43,3 @@ $(RUN_SCRIPT):: $(foreach id,$(ENABLE_HOST_UARTID),echo '$(call RunHostTerminal,$(id))' >> $@;) \ echo '$(TARGET_OUT_0) $(CLI_TARGET_OPTIONS) -- $(HOST_PARAMETERS)' >> $@; \ chmod a+x $@ - -.PHONY: flash -flash: all flashfs flashpart ##Write all images to (virtual) flash diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index ce22e5e331..ac421a26e8 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -8,14 +8,20 @@ override HWCONFIG := standard $(info Using configuration '$(HWCONFIG)') endif +# Directories to search for hardware config HWCONFIG_DIRS := $(PROJECT_DIR) $(COMPONENT_SEARCH_DIRS) $(ARCH_BASE) $(SMING_HOME) -HWCONFIG_PATH := $(firstword $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/$(HWCONFIG).hw))) -ALL_HWCONFIG := $(sort $(notdir $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/*.hw)))) + +# List of all hardware configurations +ALL_HWCONFIG_PATHS := $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/*.hw)) +ALL_HWCONFIG := $(sort $(notdir $(ALL_HWCONFIG_PATHS))) ALL_HWCONFIG := $(ALL_HWCONFIG:.hw=) +# Path to selected hardware config file +HWCONFIG_PATH := $(firstword $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/$(HWCONFIG).hw))) + ifeq (,$(wildcard $(HWCONFIG_PATH))) $(info $(HWCONFIG_DIRS)) -$(eval $(call PrintVariable,ALL_HWCONFIG)) +$(eval $(call PrintVariable,ALL_HWCONFIG,Available configurations)) $(error Hardware configuration '$(HWCONFIG)' not found) endif @@ -28,6 +34,11 @@ HWCONFIG_TOOL := \ ifeq (,$(MAKE_DOCS)) +# Generate build variables from hardware configuration +HWCONFIG_MK := $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk +$(shell $(HWCONFIG_TOOL) --quiet expr $(HWCONFIG) $(HWCONFIG_MK) "config.buildVars()") +include $(HWCONFIG_MK) + HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part $(PART)) expr $(HWCONFIG) - define HwExpr @@ -45,8 +56,10 @@ endef # Import PARTITION_TABLE_OFFSET from hardware configuration DEBUG_VARS += PARTITION_TABLE_OFFSET SMING_ARCH_HW := $(call HwExpr,config.arch) -ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) -$(error Hardware configuration '$(HWCONFIG)' is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) +ifeq ($(SMING_ARCH_HW),) +$(error Hardware configuration error) +else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) +$(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) endif PARTITION_TABLE_OFFSET := $(call HwExpr,(config.partitions.offset_str())) COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) @@ -55,8 +68,10 @@ COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) ##@Configuration .PHONY: map -map: $(HWCONFIG_PATH) ##Show memory map +map: $(HWCONFIG_PATH) ##Show partition map + @echo "Partition map:" $(Q) $(HWEXPR) "config.map().to_csv()" + @echo .PHONY: hwexpr hwexpr: $(HWCONFIG_PATH) ##Evaluate expression against hardware configuration (use EXPR= and optionally PART=) @@ -65,16 +80,23 @@ hwexpr: $(HWCONFIG_PATH) ##Evaluate expression against hardware configuration (u .PHONY: hwconfig hwconfig: $(HWCONFIG_PATH) ##Show current hardware configuration @echo - @echo Hardware Configuration - @echo Available: $(ALL_HWCONFIG) - @echo Selected: $(HWCONFIG) $(Q) $(HWEXPR) "config.to_json()" @echo +ifneq ($(V),) + @echo "$(HWCONFIG): $(foreach c,$(HWCONFIG_DEPENDS),\n $c)" +endif + +.PHONY: hwconfig-list +hwconfig-list: ##List available hardware configurations + @echo "Available configurations: $(foreach c,$(ALL_HWCONFIG),\n $(if $(subst $c,,$(HWCONFIG)), ,*) $(shell printf "%-25s" "$c") $(filter %/$c.hw,$(ALL_HWCONFIG_PATHS)))" + @echo -.PHONY: hwconfig-create -hwconfig-create: ##Create hardware configuration from current build variables - $(Q) $(HWCONFIG_TOOL) create-config $(CONFIG_CACHE_FILE) - +# @echo "Available configurations: $(foreach c,$(ALL_HWCONFIG_PATHS),\n $(if $(subst $c,,$(HWCONFIG)), ,*) $c)" +.PHONY: hwconfig-validate +hwconfig-validate: $(HWCONFIG_PATH) ##Validate current hardware configuration + @echo "Validating hardware config '$(HWCONFIG)'" + $(Q) $(HWCONFIG_TOOL) validate $(HWCONFIG) - $(PARTITION_PATH)/schema.json ##@Building @@ -82,28 +104,79 @@ hwconfig-create: ##Create hardware configuration from current build variables PARTITIONS_BIN := $(FW_BASE)/partitions.bin CUSTOM_TARGETS += $(PARTITIONS_BIN) -$(PARTITIONS_BIN): $(HWCONFIG_PATH) +$(PARTITIONS_BIN): $(HWCONFIG_DEPENDS) + $(Q) $(MAKE) --no-print-directory hwconfig-validate $(Q) $(HWCONFIG_TOOL) partgen $(HWCONFIG) $@ +# Create build target for a partition +# $1 -> Partition name +define PartitionTarget +PTARG := $(shell $(HWCONFIG_TOOL) --part $1 expr $(HWCONFIG) - part.filename) +$$(PTARG): + $$(Q) $$(MAKE) --no-print-directory $$(shell $$(HWCONFIG_TOOL) --part $1 expr $$(HWCONFIG) - "part.build['target']") PART=$1 +CUSTOM_TARGETS += $$(PTARG) +endef + +# Create build targets for all partitions with 'make' entry +DEBUG_VARS += PARTITIONS_WITH_TARGETS +PARTITIONS_WITH_TARGETS := $(call HwExpr,(' '.join([part.name for part in filter(lambda part: part.build is not None, config.partitions)]))) + +# Must be invoked from project.mk after all Components have been processed +# This allows partition definitions to include variables which may not yet be defined +define PartitionCreateTargets +$(foreach p,$(PARTITIONS_WITH_TARGETS),$(eval $(call PartitionTarget,$p))) +endef + + ##@Flashing # $1 -> Filter expression define PartChunks -$(call HwExpr,(' '.join(['%s=%s' % (part.address_str(), part.filename) for part in filter(lambda part: $1 and part.filename != '', config.partitions)]))) +$(call HwExpr,(' '.join(['%s=%s' % (part.address_str(), part.filename) for part in filter(lambda part: $1 and part.device.name == 'spiFlash' and part.filename != '', config.partitions)]))) endef # One flash sector of 0xFF BLANK_BIN := $(PARTITION_PATH)/blank.bin -DEBUG_VARS += FLASH_APP_CHUNKS FLASH_PARTITION_CHUNKS +DEBUG_VARS += FLASH_APP_CHUNKS FLASH_MAP_CHUNKS FLASH_PARTITION_CHUNKS $(eval FLASH_APP_CHUNKS = $(call PartChunks,(part.type_is('app')))) -$(eval FLASH_PARTITION_CHUNKS = $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) $(call PartChunks,True)) +FLASH_MAP_CHUNK := $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) +$(eval FLASH_PARTITION_CHUNKS = $(patsubst %=,,$(call PartChunks,True))) ifdef PART -DEBUG_VARS += FLASH_PART_CHUNKS -$(eval FLASH_PART_CHUNKS = $(call PartChunks,(part=='$(PART)'))) +DEBUG_VARS += FLASH_PART_CHUNKS FLASH_PART_REGION +$(eval FLASH_PART_CHUNKS := $(call PartChunks,(part=='$(PART)'))) +$(eval FLASH_PART_REGION := $(call HwExpr,('%s,%s' % (part.address_str(), part.size)))) endif +# Where to store read partition map file +READMAP_BIN := $(OUT_BASE)/partition-table.read.bin +PARTITION_TABLE_REGION := $(PARTITION_TABLE_OFFSET),0x0c00 + +.PHONY: readmap +readmap:##Read partition map from device + $(Q) $(call ReadFlash,$(PARTITION_TABLE_REGION),$(READMAP_BIN)) + @echo + @echo "Partition map read from device:" + $(Q) $(HWCONFIG_TOOL) expr $(READMAP_BIN) - "config.map().to_csv()" + @echo + +.PHONY: flashpart +flashpart: all kill_term ##Flash a specific partition, set PART=name + $(call WriteFlash,$(FLASH_PART_CHUNKS)) + +.PHONY: erasepart +erasepart: kill_term ##Erase a specific partition, set PART=name + $(call EraseFlashRegion,$(FLASH_PART_REGION)) + +.PHONY: readpart +readpart: kill_term ##Read partition from device, set PART=name + $(call ReadFlash,$(FLASH_PART_REGION),$(OUT_BASE)/$(PART).read.bin) + +.PHONY: flashmap +flashmap: $(PARTITIONS_BIN) kill_term ##Write partition table to device + $(call WriteFlash,$(FLASH_MAP_CHUNK)) + endif # MAKE_DOCS diff --git a/Sming/Components/esptool/README.rst b/Sming/Components/esptool/README.rst index df11b055fa..43acb7c806 100644 --- a/Sming/Components/esptool/README.rst +++ b/Sming/Components/esptool/README.rst @@ -8,18 +8,26 @@ Options .. envvar:: SPI_SPEED + [read-only] Set by :ref:`hardware_config`. + Clock speed for flash memory (20, 26, 40 or 80). Default is 40. .. envvar:: SPI_MODE - Flash memory operating mode (quot, dio, dout, qio). Default is qio. + [read-only] Set by :ref:`hardware_config`. + Flash memory operating mode (quot, dio, dout, qio). Default is qio. .. envvar:: SPI_SIZE + [read-only] Set by :ref:`hardware_config`. + Size of flash memory chip (256KB, 512KB, 1MB, 2MB, 4MB). Default is 512K bytes. + The default hardware profile ``standard`` sets this to 1MB. + You can set ``HWCONFIG=standard-4m`` to increase it or create a custom :ref:`hardware_config` for your project. + .. envvar:: ESPTOOL diff --git a/Sming/Components/esptool/component.mk b/Sming/Components/esptool/component.mk index f982afc1c2..4004bece99 100644 --- a/Sming/Components/esptool/component.mk +++ b/Sming/Components/esptool/component.mk @@ -1,43 +1,10 @@ COMPONENT_LIBNAME := -CONFIG_VARS += SPI_SPEED SPI_MODE SPI_SIZE +SPI_SPEED = $(STORAGE_DEVICE_spiFlash_SPEED) +SPI_MODE = $(STORAGE_DEVICE_spiFlash_MODE) +SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) -# SPI_SPEED = 40, 26, 20, 80 -SPI_SPEED ?= 40 -# SPI_MODE: qio, qout, dio, dout -SPI_MODE ?= dio -# SPI_SIZE: 512K, 256K, 1M, 2M, 4M -SPI_SIZE ?= 1M - -ifeq ($(SPI_SPEED), 26) - flashimageoptions := -ff 26m -else ifeq ($(SPI_SPEED), 20) - flashimageoptions := -ff 20m -else ifeq ($(SPI_SPEED), 80) - flashimageoptions := -ff 80m -else - flashimageoptions := -ff 40m -endif - -ifeq ($(SPI_MODE), qout) - flashimageoptions += -fm qout -else ifeq ($(SPI_MODE), dio) - flashimageoptions += -fm dio -else ifeq ($(SPI_MODE), dout) - flashimageoptions += -fm dout -else - flashimageoptions += -fm qio -endif - -# Calculate parameters from SPI_SIZE value (esptool will check validity) -ifeq ($(SPI_SIZE),detect) -flashimageoptions += -fs detect -else -flashimageoptions += -fs $(SPI_SIZE)B -endif -FLASH_SIZE := $(subst M,*1024K,$(SPI_SIZE)) -FLASH_SIZE := $(subst K,*1024,$(FLASH_SIZE)) -BLANK_BIN := $(COMPONENT_PATH)/blank.bin +flashimageoptions += -fs $(SPI_SIZE)B -ff $(SPI_SPEED)m -fm $(SPI_MODE) # Default COM port and speed used for flashing CACHE_VARS += COM_PORT_ESPTOOL COM_SPEED_ESPTOOL @@ -73,6 +40,8 @@ else ESPTOOL_EXECUTE = $(ESPTOOL_CMDLINE) $1 endif +comma := , + # Write file contents to Flash # $1 -> List of `Offset=File` chunks define WriteFlash @@ -82,6 +51,21 @@ define WriteFlash ) endef +# Read flash memory into file +# $1 -> `Offset,Size` chunk +# $2 -> Output filename +define ReadFlash + $(info ReadFlash $1,$2) + $(call ESPTOOL_EXECUTE,read_flash $(subst $(comma), ,$1) $2) +endef + +# Erase a region of Flash +# $1 -> Offset,Size +define EraseFlashRegion + $(info EraseFlashRegion $1) + $(call ESPTOOL_EXECUTE,erase_region $(subst $(comma), ,$1)) +endef + # Erase flash memory contents define EraseFlash $(call ESPTOOL_EXECUTE,erase_flash) diff --git a/Sming/Components/rboot/component.mk b/Sming/Components/rboot/component.mk index e8fb9993c4..0d7617b94e 100644 --- a/Sming/Components/rboot/component.mk +++ b/Sming/Components/rboot/component.mk @@ -35,23 +35,22 @@ $(error Cannot enable RBOOT_GPIO_ENABLED and RBOOT_GPIO_SKIP_ENABLED at the same endif ### ROM Addresses ### -# Make sure that your ROM slots and SPIFFS slot(s) do not overlap! -CONFIG_VARS += RBOOT_ROM0_ADDR RBOOT_ROM1_ADDR RBOOT_ROM2_ADDR +DEBUG_VARS += RBOOT_ROM0_ADDR RBOOT_ROM1_ADDR RBOOT_ROM2_ADDR # Location of first ROM -RBOOT_ROM0_ADDR ?= 0x008000 +RBOOT_ROM0_ADDR := $(PARTITION_rom0_ADDRESS) # The parameter below specifies the location of the second ROM. # You need a second slot for any kind of firmware update mechanism. # Leave empty if you don't need a second ROM slot. -RBOOT_ROM1_ADDR ?= +RBOOT_ROM1_ADDR := $(PARTITION_rom1_ADDRESS) # The parameter below specifies the location of the GPIO ROM. # It is only used when RBOOT_GPIO_ENABLED = 1 # Note that setting this parameter will only configure rboot. # The Sming build system does not create a ROM for this slot. -RBOOT_ROM2_ADDR ?= +RBOOT_ROM2_ADDR := $(PARTITION_rom2_ADDRESS) ifeq ($(RBOOT_GPIO_ENABLED),0) ifneq ($(RBOOT_ROM2_ADDR),) @@ -86,17 +85,6 @@ RBOOT_LD_TEMPLATE ?= $(RBOOT_DIR)/rboot.rom0.ld RBOOT_LD_0 := $(BUILD_BASE)/$(RBOOT_ROM_0).ld RBOOT_LD_1 := $(BUILD_BASE)/$(RBOOT_ROM_1).ld -# -CONFIG_VARS += RBOOT_SPIFFS_0 RBOOT_SPIFFS_1 -RBOOT_SPIFFS_0 ?= 0x100000 -RBOOT_SPIFFS_1 ?= 0x300000 -APP_CFLAGS += -DRBOOT_SPIFFS_0=$(RBOOT_SPIFFS_0) -APP_CFLAGS += -DRBOOT_SPIFFS_1=$(RBOOT_SPIFFS_1) - -ifneq ($(SPI_SIZE),1M) -SPIFF_START_ADDR ?= $(RBOOT_SPIFFS_0) -endif - # filenames and options for generating rBoot rom images with esptool2 RBOOT_E2_SECTS ?= .text .text1 .data .rodata RBOOT_E2_USER_ARGS ?= -quiet -bin -boot2 @@ -139,7 +127,9 @@ COMPONENT_CXXFLAGS += \ -DRBOOT_ROM0_ADDR=$(RBOOT_ROM0_ADDR) \ -DRBOOT_ROM1_ADDR=$(RBOOT_ROM1_ADDR) -ifndef RBOOT_EMULATION +ifdef RBOOT_EMULATION +FLASH_BOOT_CHUNKS = 0x00000=$(BLANK_BIN) +else export RBOOT_ROM0_ADDR export RBOOT_ROM1_ADDR RBOOT_BIN := $(FW_BASE)/rboot.bin @@ -159,14 +149,9 @@ endef LIBMAIN_COMMANDS += $(RBOOT_LIBMAIN_COMMANDS) endif -endif # RBOOT_EMULATION - # Define our flash chunks -FLASH_RBOOT_BOOT_CHUNKS := 0x00000=$(RBOOT_BIN) -FLASH_RBOOT_APP_CHUNKS := $(RBOOT_ROM0_ADDR)=$(RBOOT_ROM_0_BIN) -FLASH_RBOOT_ERASE_CONFIG_CHUNKS := 0x01000=$(SDK_BASE)/bin/blank.bin - -ifndef RBOOT_EMULATION +FLASH_BOOT_CHUNKS := 0x00000=$(RBOOT_BIN) +FLASH_RBOOT_ERASE_CONFIG_CHUNKS := 0x01000=$(SDK_BASE)/bin/blank.bin # => Automatic linker script generation from template # $1 -> application target diff --git a/Sming/Components/spiffs/component.mk b/Sming/Components/spiffs/component.mk index e9019a6382..dc2f497a39 100644 --- a/Sming/Components/spiffs/component.mk +++ b/Sming/Components/spiffs/component.mk @@ -19,26 +19,12 @@ CONFIG_VARS += DISABLE_SPIFFS DISABLE_SPIFFS ?= 0 APP_CFLAGS += -DDISABLE_SPIFFS=$(DISABLE_SPIFFS) +ifeq ($(DISABLE_SPIFFS),0) CACHE_VARS += SPIFF_FILES SPIFF_BIN SPIFF_FILES ?= files SPIFF_BIN ?= spiff_rom SPIFF_BIN_OUT := $(FW_BASE)/$(SPIFF_BIN).bin -CUSTOM_TARGETS += $(SPIFF_BIN_OUT) - -CONFIG_VARS += SPIFF_SIZE -ifeq ($(SPI_SIZE), 256K) - SPIFF_SIZE ?= 131072 #128K -else ifeq ($(SPI_SIZE), 1M) - SPIFF_SIZE ?= 524288 #512K -else ifeq ($(SPI_SIZE), 2M) - SPIFF_SIZE ?= 524288 #512K -else ifeq ($(SPI_SIZE), 4M) - SPIFF_SIZE ?= 524288 #512K -else - SPIFF_SIZE ?= 196608 #192K endif -APP_CFLAGS += -DSPIFF_SIZE=$(SPIFF_SIZE) - COMPONENT_RELINK_VARS += SPIFF_FILEDESC_COUNT SPIFF_FILEDESC_COUNT ?= 7 @@ -46,57 +32,20 @@ COMPONENT_CFLAGS += -DSPIFF_FILEDESC_COUNT=$(SPIFF_FILEDESC_COUNT) COMPONENT_CFLAGS += -Wno-tautological-compare - -ifeq ($(SMING_ARCH),Esp32) -SPIFF_START_ADDR ?= 0x200000 -APP_CFLAGS += -DSPIFF_SIZE=$(SPIFF_SIZE) -DSPIFF_START_ADDR=$(SPIFF_START_ADDR) -endif - ##@Building .PHONY: spiffs-image-update spiffs-image-update: spiffs-image-clean $(SPIFF_BIN_OUT) ##Rebuild the SPIFFS filesystem image -##@Cleaning - -.PHONY: spiffs-image-clean -spiffs-image-clean: ##Remove SPIFFS image file - $(info Cleaning $(SPIFF_BIN_OUT)) - $(Q) rm -f $(SPIFF_BIN_OUT) - -# Generating spiffs_bin -$(SPIFF_BIN_OUT): $(SPIFFY) -ifeq ($(DISABLE_SPIFFS), 1) - $(info (!) Spiffs support disabled. Remove 'DISABLE_SPIFFS' make argument to enable spiffs.) -else - $(Q) mkdir -p $(dir $(SPIFF_BIN_OUT)) - $(info Checking for spiffs files) - $(Q) if [ -d "$(SPIFF_FILES)" ]; then \ - echo "$(SPIFF_FILES) directory exists. Creating $(SPIFF_BIN_OUT)"; \ - $(SPIFFY) $(SPIFF_SIZE) $(SPIFF_FILES) $(SPIFF_BIN_OUT); \ - else \ - echo "No files found in ./$(SPIFF_FILES)."; \ - echo "Creating empty $(SPIFF_BIN_OUT)"; \ - $(SPIFFY) $(SPIFF_SIZE) dummy.dir $(SPIFF_BIN_OUT); \ - fi -endif - -##@Flashing - -BLANKFS_BIN := $(COMPONENT_PATH)/blankfs.bin - -# If enabled, add the SPIFFS image to the chunks to write -ifeq ($(DISABLE_SPIFFS), 1) -FLASH_SPIFFS_CHUNKS := -else -FLASH_SPIFFS_CHUNKS := $(SPIFF_START_ADDR)=$(SPIFF_BIN_OUT) -FLASH_INIT_CHUNKS += $(SPIFF_START_ADDR)=$(BLANKFS_BIN) +# Target invoked via partition table +ifneq (,$(filter spiffsgen,$(MAKECMDGOALS))) +PART_TARGET := $(PARTITION_$(PART)_FILENAME) +$(eval PART_FILES := $(call HwExpr,part.build['files'])) +SPIFFY_ARGS := $(PARTITION_$(PART)_SIZE_BYTES) "$(or $(PART_FILES),dummy.dir)" +.PHONY: spiffsgen +spiffsgen: $(SPIFFY) +ifneq (,$(PART_TARGET)) + $(Q) mkdir -p $(dir $(PART_TARGET)) + $(Q) $(SPIFFY) $(SPIFFY_ARGS) $(PART_TARGET) endif - -.PHONY: flashfs -flashfs: $(SPIFF_BIN_OUT) ##Write just the SPIFFS filesystem image -ifeq ($(DISABLE_SPIFFS), 1) - $(info SPIFFS image creation disabled!) -else - $(call WriteFlash,$(FLASH_SPIFFS_CHUNKS)) endif diff --git a/Sming/component-wrapper.mk b/Sming/component-wrapper.mk index 81af7e389e..ae7d2c4cfd 100644 --- a/Sming/component-wrapper.mk +++ b/Sming/component-wrapper.mk @@ -15,6 +15,7 @@ all: $(error Internal makefile) include $(SMING_HOME)/build.mk +include $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk # Makefile runs in the build directory COMPONENT_BUILD_DIR := $(CURDIR) diff --git a/Sming/component.mk b/Sming/component.mk index 05acf63028..c0fae08108 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -148,8 +148,25 @@ GLOBAL_CFLAGS += -DSTRING_OBJECT_SIZE=$(STRING_OBJECT_SIZE) ##@Flashing .PHONY: flashinit -flashinit: $(ESPTOOL) $(FLASH_INIT_DATA) | $(FW_BASE) ##Erase your device's flash memory and reset system configuration area to defaults +flashinit: $(ESPTOOL) $(FLASH_INIT_DATA) | $(FW_BASE) ##Erase your device's flash memory $(info Flash init data default and blank data) $(info DISABLE_SPIFFS = $(DISABLE_SPIFFS)) $(Q) $(EraseFlash) $(call WriteFlash,$(FLASH_INIT_CHUNKS)) + +.PHONY: flashboot +flashboot: $(FLASH_BOOT_LOADER) kill_term ##Write just the Bootloader + $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) + +.PHONY: flashapp +flashapp: all kill_term ##Write just the application image + $(call WriteFlash,$(FLASH_APP_CHUNKS)) + +.PHONY: flash +flash: all kill_term ##Write the boot loader and all defined partition images + $(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_MAP_CHUNK) $(FLASH_PARTITION_CHUNKS)) +ifeq ($(ENABLE_GDB), 1) + $(GDB_CMDLINE) +else ifneq ($(SMING_ARCH),Host) + $(TERMINAL) +endif diff --git a/Sming/project.mk b/Sming/project.mk index bc3f64b678..91b7db27df 100644 --- a/Sming/project.mk +++ b/Sming/project.mk @@ -319,6 +319,8 @@ CMP_App_VARS := $(CONFIG_VARS) CMP_App_ALL_VARS := $(CONFIG_VARS) $(foreach c,$(COMPONENTS),$(eval $(call ParseComponentLibs,$c))) +$(PartitionCreateTargets) + export COMPONENTS_EXTRA_INCDIR export APPCODE export APP_CFLAGS From 6a07e9caa22c226fe86049606595fa63f935accb Mon Sep 17 00:00:00 2001 From: mikee47 Date: Tue, 8 Dec 2020 23:34:45 +0000 Subject: [PATCH 17/36] Update samples Set HWCONFIG if required (SPIFFS or 4M flash) Flash size for 'standard' is small so it runs on all devices --- samples/Arducam/component.mk | 2 +- samples/Basic_Blink/component.mk | 11 ++++++++--- samples/Basic_Serial/component.mk | 3 ++- samples/Basic_Ssl/component.mk | 3 --- samples/Basic_WebSkeletonApp/component.mk | 5 +---- samples/Basic_WebSkeletonApp_LTS/component.mk | 3 ++- samples/CommandProcessing_Debug/component.mk | 2 +- samples/Echo_Ssl/component.mk | 5 ----- samples/FtpServer_Files/component.mk | 2 ++ samples/HttpClient/component.mk | 3 +-- samples/HttpClient_Instapush/component.mk | 1 + samples/HttpClient_ThingSpeak/component.mk | 1 + samples/HttpServer_AJAX/component.mk | 1 + samples/HttpServer_Bootstrap/component.mk | 2 ++ samples/HttpServer_ConfigNetwork/component.mk | 1 + samples/HttpServer_FirmwareUpload/README.rst | 10 +--------- samples/HttpServer_FirmwareUpload/component.mk | 1 + samples/HttpServer_WebSockets/component.mk | 1 + samples/MeteoControl/component.mk | 1 + samples/MeteoControl_mqtt/component.mk | 2 -- samples/MqttClient_Hello/component.mk | 5 +---- samples/ScreenTFT_ILI9340-ILI9341/component.mk | 1 + samples/ScreenTFT_ST7735/component.mk | 1 + samples/SmtpClient/component.mk | 1 + samples/UdpServer_mDNS/component.mk | 1 + samples/Websocket_Client/component.mk | 5 +---- 26 files changed, 34 insertions(+), 40 deletions(-) create mode 100644 samples/FtpServer_Files/component.mk create mode 100644 samples/HttpServer_Bootstrap/component.mk create mode 100644 samples/HttpServer_WebSockets/component.mk diff --git a/samples/Arducam/component.mk b/samples/Arducam/component.mk index 3d7a1074ba..961e2d7ea9 100644 --- a/samples/Arducam/component.mk +++ b/samples/Arducam/component.mk @@ -1,3 +1,3 @@ ARDUINO_LIBRARIES := ArduCAM - +HWCONFIG = spiffs SPIFF_FILES = web/build diff --git a/samples/Basic_Blink/component.mk b/samples/Basic_Blink/component.mk index ec6697542e..aeac5ea204 100644 --- a/samples/Basic_Blink/component.mk +++ b/samples/Basic_Blink/component.mk @@ -29,11 +29,16 @@ # COMPONENT_CFLAGS := # COMPONENT_CXXFLAGS := -## Configure flash parameters (for ESP12-E and other new boards): -# SPI_MODE := dio +## Configure hardware +# Default is 'standard' (no spiffs), can also provide your own +#HWCONFIG := spiffs -## SPIFFS options +## SPIFFS options (deprecated: use HWCONFIG) +## 0: (default) use `HWCONFIG=standard` as default +## 1: Use `HWCONFIG=spiffs` DISABLE_SPIFFS := 1 + +## Select source of content for default `spiffs` partition when built # SPIFF_FILES = files ## Refer to Basic_rBoot sample for options relating to rBoot diff --git a/samples/Basic_Serial/component.mk b/samples/Basic_Serial/component.mk index ecfe972d0a..96def499d3 100644 --- a/samples/Basic_Serial/component.mk +++ b/samples/Basic_Serial/component.mk @@ -1,4 +1,4 @@ -SPIFF_SIZE ?= 65536 +HWCONFIG := spiffs CUSTOM_TARGETS += files/README.md @@ -10,3 +10,4 @@ files/README.md: $(SMING_HOME)/../README.md # Emulate both serial ports ENABLE_HOST_UARTID := 0 1 +HOST_NETWORK_OPTIONS := --pause diff --git a/samples/Basic_Ssl/component.mk b/samples/Basic_Ssl/component.mk index ad42913baa..089769b112 100644 --- a/samples/Basic_Ssl/component.mk +++ b/samples/Basic_Ssl/component.mk @@ -6,7 +6,4 @@ COMPONENT_DEPENDS += malloc_count COMPONENT_CXXFLAGS += -DENABLE_MALLOC_COUNT=1 endif -## size of the flash chip -SPI_SIZE ?= 4M - ENABLE_SSL = 1 diff --git a/samples/Basic_WebSkeletonApp/component.mk b/samples/Basic_WebSkeletonApp/component.mk index 590d42b363..c6982bde3d 100644 --- a/samples/Basic_WebSkeletonApp/component.mk +++ b/samples/Basic_WebSkeletonApp/component.mk @@ -1,5 +1,2 @@ ARDUINO_LIBRARIES := ArduinoJson6 - -SPIFF_SIZE ?= 196608 - -DISABLE_SPIFFS ?= 0 +HWCONFIG := spiffs diff --git a/samples/Basic_WebSkeletonApp_LTS/component.mk b/samples/Basic_WebSkeletonApp_LTS/component.mk index e00a805aa3..8621049159 100644 --- a/samples/Basic_WebSkeletonApp_LTS/component.mk +++ b/samples/Basic_WebSkeletonApp_LTS/component.mk @@ -26,7 +26,8 @@ # Com port speed # COM_SPEED = 115200 -SPIFF_SIZE = 196608 +# +HWCONFIG = spiffs # ARDUINO_LIBRARIES := OneWire ArduinoJson5 diff --git a/samples/CommandProcessing_Debug/component.mk b/samples/CommandProcessing_Debug/component.mk index 973dfe71cf..298d44e80b 100644 --- a/samples/CommandProcessing_Debug/component.mk +++ b/samples/CommandProcessing_Debug/component.mk @@ -1 +1 @@ -DISABLE_SPIFFS = 1 +HWCONFIG := spiffs diff --git a/samples/Echo_Ssl/component.mk b/samples/Echo_Ssl/component.mk index 1e5eedd2a0..ae82bad52a 100644 --- a/samples/Echo_Ssl/component.mk +++ b/samples/Echo_Ssl/component.mk @@ -1,6 +1 @@ -DISABLE_SPIFFS = 1 - -## size of the flash chip -SPI_SIZE ?= 4M - ENABLE_SSL = 1 diff --git a/samples/FtpServer_Files/component.mk b/samples/FtpServer_Files/component.mk new file mode 100644 index 0000000000..1014c23996 --- /dev/null +++ b/samples/FtpServer_Files/component.mk @@ -0,0 +1,2 @@ +HWCONFIG := spiffs +DISABLE_SPIFFS := 1 diff --git a/samples/HttpClient/component.mk b/samples/HttpClient/component.mk index f3ee8603fd..b5818865f6 100644 --- a/samples/HttpClient/component.mk +++ b/samples/HttpClient/component.mk @@ -1,5 +1,4 @@ -## size of the flash chip -SPI_SIZE ?= 4M +HWCONFIG = spiffs ## Prefer BearSSL as it can handle more gracefully big SSL packets. ENABLE_SSL ?= Bearssl diff --git a/samples/HttpClient_Instapush/component.mk b/samples/HttpClient_Instapush/component.mk index c304e7f8f4..478f05574c 100644 --- a/samples/HttpClient_Instapush/component.mk +++ b/samples/HttpClient_Instapush/component.mk @@ -1,2 +1,3 @@ +HWCONFIG := spiffs DISABLE_SPIFFS = 1 ARDUINO_LIBRARIES := ArduinoJson6 diff --git a/samples/HttpClient_ThingSpeak/component.mk b/samples/HttpClient_ThingSpeak/component.mk index 973dfe71cf..d4730a5d9f 100644 --- a/samples/HttpClient_ThingSpeak/component.mk +++ b/samples/HttpClient_ThingSpeak/component.mk @@ -1 +1,2 @@ +HWCONFIG := spiffs DISABLE_SPIFFS = 1 diff --git a/samples/HttpServer_AJAX/component.mk b/samples/HttpServer_AJAX/component.mk index 6e7cdbc0c9..50337d80c6 100644 --- a/samples/HttpServer_AJAX/component.mk +++ b/samples/HttpServer_AJAX/component.mk @@ -1,2 +1,3 @@ +HWCONFIG := spiffs SPIFF_FILES = web/build ARDUINO_LIBRARIES := ArduinoJson6 diff --git a/samples/HttpServer_Bootstrap/component.mk b/samples/HttpServer_Bootstrap/component.mk new file mode 100644 index 0000000000..1014c23996 --- /dev/null +++ b/samples/HttpServer_Bootstrap/component.mk @@ -0,0 +1,2 @@ +HWCONFIG := spiffs +DISABLE_SPIFFS := 1 diff --git a/samples/HttpServer_ConfigNetwork/component.mk b/samples/HttpServer_ConfigNetwork/component.mk index ef3cde28e8..2574c69f39 100644 --- a/samples/HttpServer_ConfigNetwork/component.mk +++ b/samples/HttpServer_ConfigNetwork/component.mk @@ -1,3 +1,4 @@ +HWCONFIG := spiffs SPIFF_FILES = web/build ARDUINO_LIBRARIES := ArduinoJson6 diff --git a/samples/HttpServer_FirmwareUpload/README.rst b/samples/HttpServer_FirmwareUpload/README.rst index c483906a07..1a6be3e966 100644 --- a/samples/HttpServer_FirmwareUpload/README.rst +++ b/samples/HttpServer_FirmwareUpload/README.rst @@ -46,15 +46,7 @@ documentation for further advice on how to use the security features properly. Usage instructions ------------------ - -1. Configure your flash memory layout: - - - Set :envvar:`SPI_SIZE` to the flash memory size of your device. - - If necessary, modify :envvar:`RBOOT_ROM0_ADDR`, :envvar:`RBOOT_ROM1_ADDR`, - :envvar:`RBOOT_SPIFFS_0` and :envvar:`SPIFF_SIZE` to fit both ROM slots and - the file system into the available flash memory. Make sure that the - flash areas do not overlap with each other or any the reserved regions. - Refer to the :component:`rboot` documentation for further details. +1. Configure your flash memory layout. See :ref:`hardware_config`. 2. Build the example by running:: diff --git a/samples/HttpServer_FirmwareUpload/component.mk b/samples/HttpServer_FirmwareUpload/component.mk index 66466ad9b0..5bba481389 100644 --- a/samples/HttpServer_FirmwareUpload/component.mk +++ b/samples/HttpServer_FirmwareUpload/component.mk @@ -1,3 +1,4 @@ +HWCONFIG := spiffs SPIFF_FILES = web/ ARDUINO_LIBRARIES := MultipartParser OtaUpgrade diff --git a/samples/HttpServer_WebSockets/component.mk b/samples/HttpServer_WebSockets/component.mk new file mode 100644 index 0000000000..298d44e80b --- /dev/null +++ b/samples/HttpServer_WebSockets/component.mk @@ -0,0 +1 @@ +HWCONFIG := spiffs diff --git a/samples/MeteoControl/component.mk b/samples/MeteoControl/component.mk index 253cf6cb30..b847e6d1b0 100644 --- a/samples/MeteoControl/component.mk +++ b/samples/MeteoControl/component.mk @@ -1,3 +1,4 @@ ARDUINO_LIBRARIES := LiquidCrystal DHTesp ArduinoJson6 +HWCONFIG := spiffs SPIFF_FILES = web diff --git a/samples/MeteoControl_mqtt/component.mk b/samples/MeteoControl_mqtt/component.mk index bdb4b87fa1..5b00021804 100644 --- a/samples/MeteoControl_mqtt/component.mk +++ b/samples/MeteoControl_mqtt/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := BMP180 SI7021 - -DISABLE_SPIFFS = 1 diff --git a/samples/MqttClient_Hello/component.mk b/samples/MqttClient_Hello/component.mk index 0b3208d9a7..10ef45cc95 100644 --- a/samples/MqttClient_Hello/component.mk +++ b/samples/MqttClient_Hello/component.mk @@ -1,8 +1,5 @@ -DISABLE_SPIFFS = 1 +HWCONFIG := standard-4m ifdef MQTT_URL USER_CFLAGS += -DMQTT_URL=\"$(MQTT_URL)\" endif - -## size of the flash chip -SPI_SIZE ?= 4M diff --git a/samples/ScreenTFT_ILI9340-ILI9341/component.mk b/samples/ScreenTFT_ILI9340-ILI9341/component.mk index 77706c8ff2..29bda5a46a 100644 --- a/samples/ScreenTFT_ILI9340-ILI9341/component.mk +++ b/samples/ScreenTFT_ILI9340-ILI9341/component.mk @@ -1 +1,2 @@ +HWCONFIG := spiffs ARDUINO_LIBRARIES := Adafruit_ILI9341 diff --git a/samples/ScreenTFT_ST7735/component.mk b/samples/ScreenTFT_ST7735/component.mk index 444739bbdd..cb9740b8f7 100644 --- a/samples/ScreenTFT_ST7735/component.mk +++ b/samples/ScreenTFT_ST7735/component.mk @@ -1 +1,2 @@ +HWCONFIG := spiffs ARDUINO_LIBRARIES := Adafruit_ST7735 diff --git a/samples/SmtpClient/component.mk b/samples/SmtpClient/component.mk index 738d384dcc..943f6630c9 100644 --- a/samples/SmtpClient/component.mk +++ b/samples/SmtpClient/component.mk @@ -1 +1,2 @@ +HWCONFIG := spiffs ENABLE_SSL ?= 1 diff --git a/samples/UdpServer_mDNS/component.mk b/samples/UdpServer_mDNS/component.mk index 75719a495d..ddb062c800 100644 --- a/samples/UdpServer_mDNS/component.mk +++ b/samples/UdpServer_mDNS/component.mk @@ -1,2 +1,3 @@ +HWCONFIG := spiffs DISABLE_SPIFFS = 1 COMPONENT_DEPENDS += mdns diff --git a/samples/Websocket_Client/component.mk b/samples/Websocket_Client/component.mk index a2dcdb1632..38bc642725 100644 --- a/samples/Websocket_Client/component.mk +++ b/samples/Websocket_Client/component.mk @@ -1,7 +1,4 @@ -DISABLE_SPIFFS = 1 +HWCONFIG := standard-4m # Uncomment the option below if you want SSL support #ENABLE_SSL=1 - -## size of the flash chip -SPI_SIZE ?= 4M From b148698f04d40afd1744017c5fbd36d9799f083f Mon Sep 17 00:00:00 2001 From: mikee47 Date: Wed, 3 Feb 2021 21:00:34 +0000 Subject: [PATCH 18/36] Fix compiler warning in ArduCAM library --- Sming/Libraries/ArduCAM/ArduCAMStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sming/Libraries/ArduCAM/ArduCAMStream.cpp b/Sming/Libraries/ArduCAM/ArduCAMStream.cpp index 91564aa79c..11a491cc14 100644 --- a/Sming/Libraries/ArduCAM/ArduCAMStream.cpp +++ b/Sming/Libraries/ArduCAM/ArduCAMStream.cpp @@ -15,7 +15,7 @@ #define BMPIMAGEOFFSET 66 -const char bmp_header[BMPIMAGEOFFSET] = +const uint8_t bmp_header[BMPIMAGEOFFSET] = { 0x42, 0x4D, 0x36, 0x58, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x00, 0x00, 0x00, 0x28, 0x00, 0x00, 0x00, 0x40, 0x01, 0x00, 0x00, 0xF0, 0x00, 0x00, 0x00, 0x01, 0x00, 0x10, 0x00, 0x03, 0x00, From e1adf68e1b5d0642b6457ee523671194cdde7fb2 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 23 Jan 2021 20:22:53 +0000 Subject: [PATCH 19/36] Add Basic_Storage sample --- samples/Basic_Storage/.cproject | 153 ++++++++++++++++++ samples/Basic_Storage/.project | 28 ++++ samples/Basic_Storage/Makefile | 9 ++ samples/Basic_Storage/README.rst | 7 + samples/Basic_Storage/app/application.cpp | 99 ++++++++++++ samples/Basic_Storage/basic_storage.hw | 55 +++++++ samples/Basic_Storage/component.mk | 4 + .../files/spiffs0/file for spiffs0.txt | 7 + .../files/spiffs1/file for spiffs1.txt | 7 + .../files/spiffs2/file for spiffs2.txt | 7 + samples/Basic_Storage/user0.bin | 1 + 11 files changed, 377 insertions(+) create mode 100644 samples/Basic_Storage/.cproject create mode 100644 samples/Basic_Storage/.project create mode 100644 samples/Basic_Storage/Makefile create mode 100644 samples/Basic_Storage/README.rst create mode 100644 samples/Basic_Storage/app/application.cpp create mode 100644 samples/Basic_Storage/basic_storage.hw create mode 100644 samples/Basic_Storage/component.mk create mode 100644 samples/Basic_Storage/files/spiffs0/file for spiffs0.txt create mode 100644 samples/Basic_Storage/files/spiffs1/file for spiffs1.txt create mode 100644 samples/Basic_Storage/files/spiffs2/file for spiffs2.txt create mode 100644 samples/Basic_Storage/user0.bin diff --git a/samples/Basic_Storage/.cproject b/samples/Basic_Storage/.cproject new file mode 100644 index 0000000000..1c32c9a8a2 --- /dev/null +++ b/samples/Basic_Storage/.cproject @@ -0,0 +1,153 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + make + -f ${ProjDirPath}/Makefile + all + true + true + true + + + make + -f ${ProjDirPath}/Makefile + clean + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flash + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashonefile + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashinit + true + true + true + + + make + -f ${ProjDirPath}/Makefile + flashboot + true + true + true + + + make + -f ${ProjDirPath}/Makefile + rebuild + true + true + true + + + + + + + + + + + + + + + + + + + + diff --git a/samples/Basic_Storage/.project b/samples/Basic_Storage/.project new file mode 100644 index 0000000000..96b9a5cf41 --- /dev/null +++ b/samples/Basic_Storage/.project @@ -0,0 +1,28 @@ + + + Basic_Storage + + + Sming + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/samples/Basic_Storage/Makefile b/samples/Basic_Storage/Makefile new file mode 100644 index 0000000000..ff51b6c3a7 --- /dev/null +++ b/samples/Basic_Storage/Makefile @@ -0,0 +1,9 @@ +##################################################################### +#### Please don't change this file. Use component.mk instead #### +##################################################################### + +ifndef SMING_HOME +$(error SMING_HOME is not set: please configure it as an environment variable) +endif + +include $(SMING_HOME)/project.mk diff --git a/samples/Basic_Storage/README.rst b/samples/Basic_Storage/README.rst new file mode 100644 index 0000000000..873299a0fd --- /dev/null +++ b/samples/Basic_Storage/README.rst @@ -0,0 +1,7 @@ +Basic Storage +============= + +This sample application demonstrates various ways to manage and access flash memory +using the :component:`Storage` system. + +It also shows how to create and partition custom storage devices. diff --git a/samples/Basic_Storage/app/application.cpp b/samples/Basic_Storage/app/application.cpp new file mode 100644 index 0000000000..97c40d6eb4 --- /dev/null +++ b/samples/Basic_Storage/app/application.cpp @@ -0,0 +1,99 @@ +#include +#include +#include +#include + +IMPORT_FSTR(FS_app, PROJECT_DIR "/app/application.cpp") + +void listSpiffsPartitions() +{ + Serial.println(_F("** Enumerate registered SPIFFS partitions")); + for(auto it = Storage::findPartition(Storage::Partition::SubType::Data::spiffs); it; ++it) { + Serial.print(F(">> Mounting '")); + Serial.print((*it).name()); + Serial.println("' ..."); + bool ok = spiffs_mount(*it); + Serial.println(ok ? "OK, listing files:" : "Mount failed!"); + if(ok) { + auto list = fileList(); + for(auto& f : fileList()) { + Serial.print(" "); + Serial.println(f); + } + Serial.print(list.count()); + Serial.println(F(" files found")); + Serial.println(); + } + } +} + +void printPart(Storage::Partition part) +{ + size_t bufSize = std::min(4096U, part.size()); + char buf[bufSize]; + OneShotFastUs timer; + if(!part.read(0, buf, bufSize)) { + debug_e("Error reading from partition '%s'", part.name().c_str()); + } else { + auto elapsed = timer.elapsedTime(); + String s = part.getDeviceName(); + s += '/'; + s += part.name(); + m_printHex(s.c_str(), buf, std::min(128U, bufSize)); + m_printf(_F("Elapsed: %s\r\n"), elapsed.toString().c_str()); + if(elapsed != 0) { + m_printf(_F("Speed: %u KB/s\r\n\r\n"), 1000 * bufSize / elapsed); + } + } + Serial.println(); +} + +void printPart(const String& partitionName) +{ + auto part = Storage::findPartition(partitionName); + if(!part) { + debug_e("Partition '%s' not found", partitionName.c_str()); + } else { + printPart(part); + } +} + +void init() +{ + Serial.begin(SERIAL_BAUD_RATE); + Serial.systemDebugOutput(true); + + listSpiffsPartitions(); + + printPart(F("user0")); + + auto part = Storage::findPartition(F("user1")); + printPart(part); + Serial.println(_F("Writing some stuff to partition...")); + String s = F("Some test stuff to write..."); + part.write(32, s.c_str(), s.length() + 1); + uint8_t buf[32]; + os_get_random(buf, sizeof buf); + part.write(64, buf, sizeof buf); + printPart(part); + + Serial.println(_F("** Reading tests, repeat 3 times to show effect of caching (if any)")); + + Serial.println(_F("** Reading SysMem device (flash)")); + part = Storage::sysMem.createPartition(F("fs_app"), FS_app, Storage::Partition::Type::data, 100); + printPart(part); + printPart(part); + printPart(part); + + Serial.println(_F("** Reading SysMem device (RAM)")); + part = Storage::sysMem.createPartition(F("fs_app"), FS_app, Storage::Partition::Type::data, 100); + printPart(part); + printPart(part); + printPart(part); + + Serial.println(_F("** Reading ProgMem device")); + part = Storage::progMem.createPartition(F("fs_app"), FS_app, Storage::Partition::Type::data, 100); + printPart(part); + printPart(part); + printPart(part); +} diff --git a/samples/Basic_Storage/basic_storage.hw b/samples/Basic_Storage/basic_storage.hw new file mode 100644 index 0000000000..44ee958b05 --- /dev/null +++ b/samples/Basic_Storage/basic_storage.hw @@ -0,0 +1,55 @@ +{ + "name": "Basic Storage sample", + "base_config": "spiffs", + "devices": { + "spiFlash": { + "mode": "qio", + "speed": 80 + } + }, + "partitions": { + "user0": { + "address": "0x1F0000", + "size": "16K", + "type": "user", + "subtype": 0, + "filename": "user0.bin" + }, + "user1": { + "address": "0x1F4000", + "size": "16K", + "type": "user", + "subtype": 1 + }, + "spiffs0": { + "address": "0x200000", + "build": { + "files": "files/spiffs0" + } + }, + "spiffs1": { + "address": "0x280000", + "size": "256K", + "type": "data", + "subtype": "spiffs", + "flags": "", + "filename": "$(FW_BASE)/spiffs1_rom.bin", + "build": { + "target": "spiffsgen", + "files": "files/spiffs1" + } + }, + "spiffs2": { + "address": "0x2C0000", + "size": "256K", + "type": "data", + "subtype": "spiffs", + "flags": "", + "filename": "$(FW_BASE)/spiffs2_rom.bin", + "build": { + "target": "spiffsgen", + "files": "files/spiffs2" + } + } + } +} \ No newline at end of file diff --git a/samples/Basic_Storage/component.mk b/samples/Basic_Storage/component.mk new file mode 100644 index 0000000000..9bb23018f7 --- /dev/null +++ b/samples/Basic_Storage/component.mk @@ -0,0 +1,4 @@ +# Use our custom hardware configuration +HWCONFIG := basic_storage + +HOST_NETWORK_OPTIONS := --nonet diff --git a/samples/Basic_Storage/files/spiffs0/file for spiffs0.txt b/samples/Basic_Storage/files/spiffs0/file for spiffs0.txt new file mode 100644 index 0000000000..1de357938e --- /dev/null +++ b/samples/Basic_Storage/files/spiffs0/file for spiffs0.txt @@ -0,0 +1,7 @@ +Basic Storage +============= + +This sample application demonstrates various ways to manage and access flash memory +using the :cpp:namespace:`Storage` API. + +It also shows how to create and partition custom storage devices. diff --git a/samples/Basic_Storage/files/spiffs1/file for spiffs1.txt b/samples/Basic_Storage/files/spiffs1/file for spiffs1.txt new file mode 100644 index 0000000000..1de357938e --- /dev/null +++ b/samples/Basic_Storage/files/spiffs1/file for spiffs1.txt @@ -0,0 +1,7 @@ +Basic Storage +============= + +This sample application demonstrates various ways to manage and access flash memory +using the :cpp:namespace:`Storage` API. + +It also shows how to create and partition custom storage devices. diff --git a/samples/Basic_Storage/files/spiffs2/file for spiffs2.txt b/samples/Basic_Storage/files/spiffs2/file for spiffs2.txt new file mode 100644 index 0000000000..1de357938e --- /dev/null +++ b/samples/Basic_Storage/files/spiffs2/file for spiffs2.txt @@ -0,0 +1,7 @@ +Basic Storage +============= + +This sample application demonstrates various ways to manage and access flash memory +using the :cpp:namespace:`Storage` API. + +It also shows how to create and partition custom storage devices. diff --git a/samples/Basic_Storage/user0.bin b/samples/Basic_Storage/user0.bin new file mode 100644 index 0000000000..2a8925f954 --- /dev/null +++ b/samples/Basic_Storage/user0.bin @@ -0,0 +1 @@ +This is some us \ No newline at end of file From b875682fe606cf92b2830357e5f0bca61f7a73d1 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 1 Jan 2021 10:28:42 +0000 Subject: [PATCH 20/36] Update docs --- Sming/Components/Storage/README.rst | 300 +++++++++++++----- Sming/Components/Storage/notes.rst | 69 ++++ Sming/building.rst | 7 + docs/Makefile | 4 +- .../arch/esp8266/getting-started/config.rst | 31 +- docs/source/information/flash.rst | 59 ++-- .../source/troubleshooting/random-restart.rst | 29 +- docs/source/upgrading/4.2-4.3.rst | 98 ++++++ docs/source/upgrading/index.rst | 1 + 9 files changed, 460 insertions(+), 138 deletions(-) create mode 100644 Sming/Components/Storage/notes.rst create mode 100644 docs/source/upgrading/4.2-4.3.rst diff --git a/Sming/Components/Storage/README.rst b/Sming/Components/Storage/README.rst index b80c0f611c..4941c6e5db 100644 --- a/Sming/Components/Storage/README.rst +++ b/Sming/Components/Storage/README.rst @@ -1,9 +1,10 @@ -Storage Manager -=============== +Storage Management +================== + +.. highlight:: bash This Component provides support for using storage devices in a structured way by partitioning them into areas for specific uses. - Partitions may can contain information such as: - Application (firmware) images @@ -12,7 +13,8 @@ Partitions may can contain information such as: - Custom flash storage areas A single partition table is located on the main flash device, :cpp:var:`Storage::spiFlash`, -and defines all partitions with a unique name and associated :cpp:type:`Storage::Partition::Type` / :cpp:type:`Storage::Partition::SubType`. +and defines all partitions with a unique name and associated +:cpp:enum:`Storage::Partition::Type` / :cpp:type:`Storage::Partition::SubType`. @@ -21,10 +23,8 @@ and defines all partitions with a unique name and associated :cpp:type:`Storage: Hardware configuration ---------------------- -Each project has an associated ``Hardware configuration``, specified by the :envvar:`HWCONFIG` setting. - -This information is defined in one or more JSON files, identified with a ``.hw`` extension. -You can find a schema for this file in :doc:`schema.json`. +Each project has an associated ``Hardware configuration``, specified by the :envvar:`HWCONFIG` setting: +this is a JSON file with a ``.hw`` extension. The build system locates the file by searching, in order: @@ -32,76 +32,192 @@ The build system locates the file by searching, in order: - ``{SMING_HOME}/Arch/{SMING_ARCH}`` - ``{SMING_HOME}`` -Files (other than standard) will have a ``base_config`` setting. The specified profile is pulled -in following the same search pattern as above. +Each architecture provides a ``standard`` configuration which defines such things as the +partition table location and standard system partitions. Other configurations inherit +from this by providing a ``base_config`` value. + +You can list the available configs like this:: + + make hwconfig-list + +This also shows the file path should you wish to view or edit it. + +To select and view the resulting configuration, do this:: + + make hwconfig HWCONFIG=spiffs + +or, to show the partition map:: + + make map HWCONFIG=spiffs + +.. note:: + + You can set :envvar:`HWCONFIG` in your project's ``component.mk`` file, however as with other + configuration variables it will be overridden by the cached value set on the command line. + + For example, if you want to change from ``standard`` to ``standard-4m`` for your project, + first add this line to your component.mk file:: + + HWCONFIG := standard-4m + + Then either run ``make HWCONFIG=standard-4m`` or ``make config-clean``. + -This provides a mechanism for inheritance to allow adjustment of existing profiles -without having to copy it entirely. +Custom configurations +--------------------- -For example, to adjust the address/size of the main application partition -(for the esp8266, this is ``rom0``), you can do this: +To customise the hardware configuration for a project, for example 'my_project': -.. code-block:: json +1. Create a new configuration file in your project root, such as ``my_project.hw``: - { - "name": "My config", - "base_config": "standard-4m", - "partitions": { - "rom0": { - "address": "0x10000", - "size": "0x80000" + .. code-block:: json + + { + "name": "My project config", + "base_config": "spiffs" + } + + You can use any available configuration as the base_config. + + +2. If required, modify any inherited settings: + + .. code-block:: json + + { + "name": "My config", + "base_config": "standard", + "devices": { + "spiFlash": { + "speed": 80, + "mode": "qio", + "size": "2M" + } + }, + "partitions": { + "rom0": { + "address": "0x10000", + "size": "0x80000" + } } } - } + This will adjust flash parameters (previously via SPI_SPEED, SPI_MODE and SPI_SIZE), + and the location/size of the primary application partition. + +3. Add any additional partitions: + + .. code-block:: json + + { + "name": "My config", + "base_config": "standard-4m", + "partitions": { + "rom0": { + "address": "0x10000", + "size": "0x80000" + }, + "spiffs1": { + "address": "0x00280000", + "size": "256K", + "type": "data", + "subtype": "spiffs", + "filename": "$(FW_BASE)/spiffs1_rom.bin", + "build": { + "target": "spiffsgen", + "files": "files/spiffs1" + } + } + } + } + + This adds a second SPIFFS partition, and instructs the build system to generate + an image file for it using the files in the project's ``files/spiffs1`` directory. + +4. Select the new configuration and re-build the project:: + + make HWCONFIG=my_project + + You should also add this to your project's ``component.mk`` file:: + + HWCONFIG := my_project -You can examine your project's active configuration like this:: +5. Program your device:: - make hwconfig + make flash -This also shows a list of the discovered configurations available. + This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry). -You can also display the memory map:: + +Partition maps +-------------- + +This is a concise view of your flash partitions. Display it like this:: make map -Which, for Host arch, produces: +For the :sample:`Basic_Storage` samnple application, we get this: .. code-block:: text - HostTests: Invoking 'map' for Host (debug) architecture - Device Start End Size Type SubType Name Filename - ---------------- -------- -------- -------- -------- -------- ---------------- ------------ - spiFlash 0x000000 0x001fff 8K Boot Sector - spiFlash 0x002000 0x002fff 4K Partition Table - spiFlash 0x003000 0x007fff 20K (unused) - spiFlash 0x008000 0x0fffff 992K app factory rom0 $(BLANK_BIN) - spiFlash 0x100000 0x1fffff 1M (unused) - spiFlash 0x200000 0x27ffff 512K data spiffs spiffs0 $(FW_BASE)/spiffs0.bin - spiFlash 0x280000 0x3fffff 1536K (unused) - test-device 0x000000 0x3fffff 4M data spiffs external1 - test-device 0x400000 0x5fffff 2M (unused) - test-device 0x600000 0x63bfff 240K data 37 external2 - test-device 0x63c000 0xffffff 10000K (unused) + Basic_Storage: Invoking 'map' for Esp8266 (debug) architecture + Partition map: + Device Start End Size Type SubType Name Filename + ---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------ + spiFlash 0x00000000 0x00001fff 8K Boot Sector + spiFlash 0x00002000 0x00002fff 4K Partition Table + spiFlash 0x00003000 0x00003fff 4K data phy phy_init $(FLASH_INIT_DATA) + spiFlash 0x00004000 0x00007fff 16K data sysparam sys_param + spiFlash 0x00008000 0x000fffff 992K app factory rom0 $(RBOOT_ROM_0_BIN) + spiFlash 0x00100000 0x001effff 960K (unused) + spiFlash 0x001f0000 0x001f3fff 16K user 0 user0 user0.bin + spiFlash 0x001f4000 0x001f7fff 16K user 1 user1 + spiFlash 0x001f8000 0x001fffff 32K (unused) + spiFlash 0x00200000 0x0027ffff 512K data spiffs spiffs0 $(SPIFF_BIN_OUT) + spiFlash 0x00280000 0x002bffff 256K data spiffs spiffs1 $(FW_BASE)/spiffs1_rom.bin + spiFlash 0x002c0000 0x002fffff 256K data spiffs spiffs2 $(FW_BASE)/spiffs2_rom.bin + spiFlash 0x00300000 0x003fffff 1M (unused) For comparison, here's the output for Esp32: .. code-block:: text - HostTests: Invoking 'map' for Esp32 (debug) architecture - Device Start End Size Type SubType Name Filename - ---------------- -------- -------- -------- -------- -------- ---------------- ------------ - spiFlash 0x000000 0x007fff 32K Boot Sector - spiFlash 0x008000 0x008fff 4K Partition Table - spiFlash 0x009000 0x00efff 24K data nvs nvs - spiFlash 0x00f000 0x00ffff 4K data phy phy_init - spiFlash 0x010000 0x1fffff 1984K app factory factory $(TARGET_BIN) - spiFlash 0x200000 0x27ffff 512K data spiffs spiffs0 $(FW_BASE)/spiffs0.bin - spiFlash 0x280000 0x3fffff 1536K (unused) - test-device 0x000000 0x3fffff 4M data spiffs external1 - test-device 0x400000 0x5fffff 2M (unused) - test-device 0x600000 0x63bfff 240K data 37 external2 - test-device 0x63c000 0xffffff 10000K (unused) + Basic_Storage: Invoking 'map' for Esp32 (debug) architecture + Partition map: + Device Start End Size Type SubType Name Filename + ---------------- ---------- ---------- ---------- -------- -------- ---------------- ------------ + spiFlash 0x00000000 0x00007fff 32K Boot Sector + spiFlash 0x00008000 0x00008fff 4K Partition Table + spiFlash 0x00009000 0x0000efff 24K data nvs nvs + spiFlash 0x0000f000 0x0000ffff 4K data phy phy_init + spiFlash 0x00010000 0x001fffff 1984K app factory factory $(TARGET_BIN) + spiFlash 0x001f0000 0x001f3fff 16K user 0 user0 user0.bin + spiFlash 0x001f4000 0x001f7fff 16K user 1 user1 + spiFlash 0x001f8000 0x001fffff 32K (unused) + spiFlash 0x00200000 0x0027ffff 512K data spiffs spiffs0 $(SPIFF_BIN_OUT) + spiFlash 0x00280000 0x002bffff 256K data spiffs spiffs1 $(FW_BASE)/spiffs1_rom.bin + spiFlash 0x002c0000 0x002fffff 256K data spiffs spiffs2 $(FW_BASE)/spiffs2_rom.bin + spiFlash 0x00300000 0x003fffff 1M (unused) + + +To compare this with the partition map programmed into a device, do this:: + + make readmap map + + +JSON validation +--------------- + +When the binary partition table is built or updated, the configuration is first +validated against a schema :source:`Sming/Components/Storage/schema.json`. + +This complements the checks performed by the ``hwconfig`` tool. + +You can run the validation manually like this:: + + make hwconfig-validate + +See `JSON Schema `__ for details about JSON schemas. Configuration @@ -111,21 +227,29 @@ Configuration default: standard - Set this to the hardware configuration file to use for your project (without .hw extension) + Set this to the hardware configuration to use for your project. Default configurations: standard - Base profile which should work on all device variants. + Base profile with 1MB flash size which should work on all device variants. Located in the ``Sming/Arch/{SMING_ARCH}`` directory. - standard-4m - Supports only devices with 4Mbyte flash - spiffs - Based on standard-4m and adds a single SPIFFS partition + + standard-4m + Overrides ``standard`` to set 4Mbyte flash size + + spiffs + Adds a single SPIFFS partition Other configurations may be available, depending on architecture. + You can see these by running ``make hwconfig-list``. + + For example, to select ``spiffs`` add the following line to your project:: + + HWCONFIG := spiffs - You can see these by running ``make hwconfig``. + You will also need to run ``make HWCONFIG=spiffs`` to change the cached value + (or ``make config-clean`` to reset everything). Binary partition table @@ -142,19 +266,11 @@ Entries are fixed 32-byte structures, :cpp:class:`Storage::esp_partition_info_t` - The partition table md5sum entry is inserted as normal - If any external devices are defined: - A SMING_EXTENSION entry, which the esp32 bootloader interprets as the end of the partition table. - - The next entry is a ``storage`` type for the 'external` device. + - The next entry is a ``storage`` type for the ``external`` device. - This is followed by regular partition entries as before. - A second md5sum entry is inserted for the entire partition table thus far - The end of the partition table is identified by an empty sector (i.e. all bytes 0xFF). -On initialisation, partition entries with the correct MAGIC entry are loaded and all others are ignored. - -A storage partition entry `address` field is always initialised to 0. -When :cpp:func:`PartitionTable::registerStorageDevice()` is called, this field is updated with -a pointer to the :cpp:class:`Storage::Device` implementation. - -Located :cpp:class:`Partition` objects contain a reference to both the requested partition -entry and the storage partition entry. Partition API @@ -162,17 +278,22 @@ Partition API This is a C++ interface. Some examples:: - auto part = partitionTable.find("spiffs0"); // Find by name + Storage::Partition part = Storage::findPartition("spiffs0"); // Find by name + if(part) { + debugf("Partition '%s' found", part.name().c_str()); + } else { + debugf("Partition NOT found"); + } // Enumerate all partitions - for(auto part: partitionTable) { + for(auto it = Storage::findPartition(); it; ++it) { + auto part = *it; debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); } // Enumerate all SPIFFS partitions - for(auto it = partitionTable.find(Partition::SubType::Data::spiffs; it; it++) { - auto part = *it; - debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + for(auto it = Storage::findPartition(Partition::SubType::Data::spiffs; it; it++) { + debugf("Found '%s' at 0x%08x, size 0x%08x", it->name().c_str(), it->address(), it->size()); } @@ -184,6 +305,31 @@ This is usually :cpp:var:`Storage::spiFlash` for the main flash device. Other devices must be registed via :cpp:func:`Storage::PartitionTable::registerStorageDevice`. +You can query partition entries from a Storage object directly, for example:: + + #include + + for(auto part: Storage::spiFlash->partitions()) { + debugf("Found '%s' at 0x%08x, size 0x%08x", part.name().c_str(), part.address(), part.size()); + } + + +External Storage +---------------- + +If your design has additional fixed storage devices, such as SPI RAM, flash or EEPROM, +you can take advantage of the partition API to manage them as follows: + +- Implement a class to manage the storage, inheriting from :cpp:class:`Storage::Device`. +- Create a custom hardware configuration for your project and add a ``devices`` entry + describing your storage device, plus partition entries: the ``device`` field identifies + which device these entries relate to. +- Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice` + in your ``init()`` function (or elsewhere if more appropriate). + + +API +--- .. doxygennamespace:: Storage :members: diff --git a/Sming/Components/Storage/notes.rst b/Sming/Components/Storage/notes.rst new file mode 100644 index 0000000000..3b4765aa06 --- /dev/null +++ b/Sming/Components/Storage/notes.rst @@ -0,0 +1,69 @@ +Storage Partition notes +======================= + +External devices +---------------- + +Support for 'external' storage devices is implemented by marking groups of entries +with a ``storage`` type partition entry. This is intended to support designs where the +memory devices are fixed (such as additional SPI RAM or flash chips) rather than +removable (such as SD cards). When these devices are registered with the Storage subsystem, +the primary partition table is scanned for related entries. + +Devices may contain their own partition table, but such tables are not loaded automatically. +This is done by calling :cpp:func:`Storage::Device::loadPartitions`. +The application must also consider how the partition table will be initialised on external +devices as this is not handled by the build system. + + + +But it also has some disadvantages: + +- Adding/removing storage devices dynamically does not fit well. + For example, when an SD Card is inserted the partition layout must be established + dynamically. + +The main purpose of the partition table is to manage the system's fixed storage. +Mostly that will just be the main flash device, but could also be 'external' devices +such as SPI RAM or flash. + +Devices can be dynamically registered/un-registered to the partition table: existing Partition objects remain valid but calls will fail. + +Dynamic partitions have additional issues. + +Partition identification +------------------------ + +Two devices may have identical partition names so these would need +to be qualified, e.g. ``spiFlash/spiffs0``. Without qualification the first matching +name would be used. + +Alternatively, each :cpp:class:`Storage::Device` object could manage its own partitions, +rather than having them in a single table. The existing layout scheme is fine, we can +just extend it so that devices can support their own partitions. + +Device serial numbers should be exposed so that devices can be uniquely identified +by applications. + +Device partitions +----------------- + +A global :cpp:class:`Storage::DeviceManager` object replaces ``partitionTable``. + +Storage each device manages its own partitions, therefore spiFlash loads its own partition table. + + + +Partitions in devices ? +----------------------- + +JSON may be more logically arranged so that partitions are defined within each device. + +This implies that partition_table_offset is also defined within the device, +so adding this entry to external devices allows definition of a partition table there also. +The application now only has to handle how to write any supplementary tables to external devices. + +Problem: Way too complicated. + +So, if we want external partition tables how to do that? +Leave for another time. \ No newline at end of file diff --git a/Sming/building.rst b/Sming/building.rst index 4904002d34..0588ec57f0 100644 --- a/Sming/building.rst +++ b/Sming/building.rst @@ -127,6 +127,13 @@ To switch to a different build architecture, for example: To inspect the current build configuration, type ``make list-config``. +Hardware configuration +~~~~~~~~~~~~~~~~~~~~~~ + +The appropriate hardware configuration should be selected in the +project's component.mk file. Use one of the standard configurations +or create your own. See :ref:`hardware-config`. + Configuration variables ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/Makefile b/docs/Makefile index 6fe6b5c76a..e8cf78e6f5 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -9,6 +9,7 @@ AWK ?= POSIXLY_CORRECT= awk SMINGDIR := .. override SMING_HOME := $(SMINGDIR)/Sming +override SMING_ARCH := Host # You can set these variables from the command line, and also from the environment for the first two. SPHINXOPTS ?= @@ -16,6 +17,7 @@ SPHINXBUILD ?= sphinx-build SOURCEDIR = source BUILDDIR = build +ARCH_BASE := $(SMING_HOME)/Arch/$(SMING_ARCH) BUILD_TYPE := release OUT_BASE := out/$(SMING_ARCH)/$(BUILD_TYPE) BUILD_BASE = $(OUT_BASE)/build @@ -115,7 +117,7 @@ DOXYGEN_PREDEFINED := \ ICACHE_FLASH_ATTR= \ __forceinline= \ SMING_DEPRECATED= \ - SMING_ARCH=Host \ + SMING_ARCH=$(SMING_ARCH) \ ARCH_HOST=1 diff --git a/docs/source/arch/esp8266/getting-started/config.rst b/docs/source/arch/esp8266/getting-started/config.rst index 920e5125d2..1798cd59d6 100644 --- a/docs/source/arch/esp8266/getting-started/config.rst +++ b/docs/source/arch/esp8266/getting-started/config.rst @@ -3,26 +3,16 @@ Configuring your Esp8266 device .. highlight:: bash -You may need to configure your project to support the specific device being programmed. +You may need to configure your project to support the specific device being programmed: -.. note:: - - Arduino has a system which allows these values to be set based on a *board* - selection. At present Sming does not have such a system so this must be done - manually. - -Here are a few important ones: - -* :envvar:`SPI_MODE` Change this if your device fails to program or run -* :envvar:`SPI_SIZE` If using a device with reduced memory you may need to change this value. Note also that samples using :component:`spiffs` may not work. * :envvar:`COM_PORT` If you haven't set this already, it will need to match the port you're using to talk to the Esp8266. * :envvar:`COM_SPEED_ESPTOOL` The default value should work fine but you can usually set a much faster speed. You can set these initially on the command line, like this:: - make SMING_ARCH=Esp8266 SPI_MODE=dio SPI_SIZE=4M COM_SPEED_ESPTOOL=921600 + make SMING_ARCH=Esp8266 COM_PORT=/dev/ttyUSB3 COM_SPEED_ESPTOOL=921600 -Once you're happy with the settings, you can add them to your ``project.mk`` file. +For Windows expect to use COM2, COM3, etc. You can list the current set of configuration variables like this:: @@ -32,14 +22,17 @@ If you want to reset back to default values, do this:: make config-clean -.. note:: +Other hardware-specific settings are stored in the hardware configuration file. +You can examine the current configuration like this:: + + make hwconfig + +The standard config should work with all ESP8266 variants. +If you want to use SPIFFS then you should add this line to your component.mk file:: - When :envvar:`SPI_MODE` or :envvar:`SPI_SIZE` are changed, you must update the on-flash configuration - data by running:: - - make flashinit + HWCONFIG = spiffs - You should do this before uploading your program code. +This expects your device to have at least 4MBytes of flash. * See :doc:`/_inc/Sming/building` for further details about configuring your project. * See :doc:`/features` for configuring Sming options. diff --git a/docs/source/information/flash.rst b/docs/source/information/flash.rst index a5a92348d7..90eb70b700 100644 --- a/docs/source/information/flash.rst +++ b/docs/source/information/flash.rst @@ -9,33 +9,50 @@ Introduction ESP8266 flash memory sizes vary from 512Kbytes on the ESP-01 up to 4Mbytes on the ESP12F. Up to 16MBytes are supported for custom designs. -You can find general details for the memory layout in the `ESP8266 Wiki `__. +Sming version 4.3 introduced partition tables to support multiple architectures, +different hardware variants and custom flash layouts without restriction. -This is the layout for Sming with a 4MByte flash device: +See :ref:`hardware-config` for details. -======= =============== ==== ========================= =================================================== -Address Config variable Size Source filename Description -(hex) (if any) (KB) (if applicable) -======= =============== ==== ========================= =================================================== -000000 1 rboot.bin Boot loader -001000 4 rBoot configuration -002000 ROM_0_ADDR rom0.bin First ROM image -100000 RBOOT_SPIFFS_0 -202000 ROM_1_ADDR rom1.bin Second ROM image -300000 RBOOT_SPIFFS_1 -3FB000 4 blank.bin RF Calibration data (Initialised to FFh) -3FC000 4 esp_init_data_default.bin PHY configuration data -3FD000 12 blank.bin System parameter area -======= =============== ==== ========================= =================================================== +A typical layout for a 4MByte device might look like this: + ======= =============== ==== ========================= =================================================== + Address Config variable Size Source filename Description + (hex) (if any) (KB) (if applicable) + ======= =============== ==== ========================= =================================================== + 000000 1 rboot.bin Boot loader + 001000 4 rBoot configuration + 002000 4 Partition table + 003000 4 esp_init_data_default.bin PHY configuration data + 004000 12 blank.bin System parameter area + 006000 4 blank.bin RF Calibration data (Initialised to FFh) + 006000 4 Reserved + 008000 ROM_0_ADDR rom0.bin First ROM image + 100000 RBOOT_SPIFFS_0 + 208000 ROM_1_ADDR rom1.bin Second ROM image + 300000 RBOOT_SPIFFS_1 + ======= =============== ==== ========================= =================================================== + -Partition Tables ----------------- +.. note:: -{ todo } + This was the previous layout for a 4MByte flash device: -Whilst SDK version 3 requires a partition table, previous versions do not but this can be added so that we -can use it as a common reference for all the above locations. + ======= =============== ==== ========================= =================================================== + Address Config variable Size Source filename Description + (hex) (if any) (KB) (if applicable) + ======= =============== ==== ========================= =================================================== + 000000 1 rboot.bin Boot loader + 001000 4 rBoot configuration + 002000 ROM_0_ADDR rom0.bin First ROM image + 100000 RBOOT_SPIFFS_0 + 202000 ROM_1_ADDR rom1.bin Second ROM image + 300000 RBOOT_SPIFFS_1 + 3FB000 4 blank.bin RF Calibration data (Initialised to FFh) + 3FC000 4 esp_init_data_default.bin PHY configuration data + 3FD000 12 blank.bin System parameter area + ======= =============== ==== ========================= =================================================== + Speed and caching diff --git a/docs/source/troubleshooting/random-restart.rst b/docs/source/troubleshooting/random-restart.rst index 818dfe9d9c..3076a189fe 100644 --- a/docs/source/troubleshooting/random-restart.rst +++ b/docs/source/troubleshooting/random-restart.rst @@ -1,9 +1,7 @@ Random Restarts =============== -First try setting the baud rate to ``74880``. Example for Linux: - -:: +First try setting the baud rate to ``74880``. Example for Linux:: python -m serial.tools.miniterm /dev/ttyUSB0 74880 @@ -15,25 +13,16 @@ device. To achieve this do the following: -1) Set the SPI_SIZE of your flash memory. Example: If you have device - with 4 megabytes make sure that the following is set: - -:: - - export SPI_SIZE=4M - -2) Run ``flashinit``. +1) Check the :ref:`hardware-config` especially ``flash_size`` setting. -Run the following commands. +2) Run ``flashinit``:: -:: + cd $SMING_HOME/../samples/Basic_Blink + make flashinit - cd $SMING_HOME/../samples/Basic_Blink - make flashinit + This resets the flash memory to a default state, erasing any existing + application, configuration or data stored there. -``flashinit`` is erasing the current flash memory and populating some -areas on it with the bytes that your SDK / BootROM is expecting to be -present. +3) Re-program your device:: -This command needs to be executed only when you change SDKs or memory -layouts. + make flash diff --git a/docs/source/upgrading/4.2-4.3.rst b/docs/source/upgrading/4.2-4.3.rst new file mode 100644 index 0000000000..9d52386086 --- /dev/null +++ b/docs/source/upgrading/4.2-4.3.rst @@ -0,0 +1,98 @@ +***************** +From v4.2 to v4.3 +***************** + +.. highlight:: c++ + + +Storage Management +------------------ + +The layout of flash memory has been revised and is now managed via partition table. +Hardware configuration is stored in a JSON file (with .hw extension). + +If your project has minimal customisation then you may only need to change +the :envvar:`HWCONFIG` setting. + +You can find full details in the :component:`Storage` library. +See also background on :doc:`/information/flash`. + +New and updated build targets + hwconfig + Displays the current configuration in JSON format + hwconfig-list + Show available hardware configs + map + Display the current flash memory map + readmap + Read partition table from device and display the map + readpart + Read contents of partition into a file, e.g. ``make readpart PART=spiffs0`` + will create ``out/Esp8266/debug/spiffs0.read.bin`` + flash + Flash partition table and all partitions. Previously it was necessary to run + ``make flashinit`` to write system parameter information. Failure to do this + was a common problem and should now be a thing of the past. + flashinit + This now just erases the flash memory, and is no longer a pre-requisite for ``make flash``. + + The ESP8266 system parameter information has been moved into registered + partitions (phy_init and sys_param) near the beginning of flash memory. + It now gets written when running ``make flash``. + + Previously, this information was kept right at the end of the flash memory, + so the location would vary depending on the setting of ``SPI_SIZE``. + This was a frequent cause of problems as the system would fail to start if this + was set incorrectly. + flashmap + Flash just the partition map + flashpart + Flash a single partition, e.g. ``make flashpart PART=spiffs0`` + erasepart + Erase a partition, e.g. ``make erasepart PART=spiffs0`` + +Configuration variables + A number of configuration variables have been removed or made read-only, as these are now + generated from the :ref:`hardware_config`. + + :envvar:`SPI_MODE`, :envvar:`SPI_SIZE`, :envvar:`SPI_SPEED` + The variables are still used internally but are read-only; they cannot be set at the command line. + Values are read from the hardware configuration under ``/devices/spiFlash``. + + :envvar:`RBOOT_ROM0_ADDR`, :envvar:`RBOOT_ROM1_ADDR`, :envvar:`RBOOT_ROM2_ADDR` + Used by :component:`rboot`, and are now read-only. + Values are read from the ``address`` property of ``rom0-2`` in the hardware configuration. + + :envvar:`RBOOT_SPIFFS_0`, :envvar:`RBOOT_SPIFFS_1` + Removed. + + :envvar:`SPIFF_SIZE` + Removed. Attempting to set this automatically within a hardware configuration is + liable to cause more problems than it solves, so updating the hardware config is + the now only way to change this setting. + + :envvar:`SPIFF_FILES` + [deprecated] + + You can still use this to specify the source location for the primary + SPIFFS partition (spiffs0). The preferred method is to set the ``files`` + property in a partition ``build`` key. + + The default SPIFFS partition settings can be overridden in a custom profile. + For example: + + .. code-block:: json + + { + ... + "base_config": "spiffs", + "partitions": { + "spiffs0": { + "size": "128K", + "build": { + "files": "some/other/folder" + } + } + } + } + diff --git a/docs/source/upgrading/index.rst b/docs/source/upgrading/index.rst index 9afa71358d..4b024707b8 100644 --- a/docs/source/upgrading/index.rst +++ b/docs/source/upgrading/index.rst @@ -7,6 +7,7 @@ For newer versions we have dedicated pages. .. toctree:: :maxdepth: 1 + 4.2-4.3 4.1-4.2 4.0-4.1 3.8-4.0 From 2e84b9c675436004c0d9571e5526c10b393cc292 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Thu, 4 Feb 2021 22:47:38 +0000 Subject: [PATCH 21/36] Fix CI & CodeQL --- .ci/install.cmd | 2 +- .ci/install.sh | 2 +- .github/workflows/codeql-analysis.yml | 10 +++------- Tools/requirements.txt | 2 ++ 4 files changed, 7 insertions(+), 9 deletions(-) create mode 100644 Tools/requirements.txt diff --git a/.ci/install.cmd b/.ci/install.cmd index 9834f28957..9dfc9811b3 100755 --- a/.ci/install.cmd +++ b/.ci/install.cmd @@ -1,6 +1,6 @@ REM Windows install script -python -m pip install --upgrade pip +python -m pip install --upgrade pip -r %SMING_HOME%\..\Tools\requirements.txt rmdir /s /q c:\MinGW curl -Lo MinGW.7z %SMINGTOOLS%/MinGW-2020-10-19.7z diff --git a/.ci/install.sh b/.ci/install.sh index 75b7fea9ca..21a90b2d2c 100755 --- a/.ci/install.sh +++ b/.ci/install.sh @@ -6,7 +6,7 @@ set -ex # exit with nonzero exit code if anything fails sudo apt-get update sudo update-alternatives --set gcc /usr/bin/gcc-9 -python -m pip install --upgrade pip +python -m pip install --upgrade pip -r $SMING_HOME/../Tools/requirements.txt sudo apt-get install -y gcc-9-multilib g++-9-multilib python3-setuptools diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 216059625b..634f26726d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -36,11 +36,6 @@ jobs: # a pull request then we can checkout the head. fetch-depth: 2 - # If this run was triggered by a pull request event, then checkout - # the head of the pull request instead of the merge commit. - - run: git checkout HEAD^2 - if: ${{ github.event_name == 'pull_request' }} - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 @@ -68,13 +63,14 @@ jobs: run: | sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test sudo apt-get update -y - sudo apt-get install gcc-9-multilib g++-9-multilib + sudo apt-get install gcc-9-multilib g++-9-multilib python3-wheel python3-setuptools sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 60 --slave /usr/bin/g++ g++ /usr/bin/g++-9 + python3 -m pip install --upgrade pip wheel -r Tools/requirements.txt env cd Sming export SMING_HOME=$(pwd) cd ../samples/Basic_Blink - make SMING_ARCH=Host + make -j3 SMING_ARCH=Host - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 diff --git a/Tools/requirements.txt b/Tools/requirements.txt new file mode 100644 index 0000000000..ee96c8f8e1 --- /dev/null +++ b/Tools/requirements.txt @@ -0,0 +1,2 @@ +pyserial +jsonschema From 7e70932eb87c39a6996549a8f3cf9d9b94df873a Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 5 Feb 2021 14:41:10 +0000 Subject: [PATCH 22/36] Tidy up Storage component.mk, fix issue with flashing where chunk filename is empty Remove redundant config queries Replace queries with variables from hwconfig.mk --- .../Storage/Tools/hwconfig/config.py | 3 +- Sming/Components/Storage/component.mk | 38 +++++++++---------- Sming/component.mk | 2 +- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Sming/Components/Storage/Tools/hwconfig/config.py b/Sming/Components/Storage/Tools/hwconfig/config.py index db42da7bb2..f0f59235c5 100644 --- a/Sming/Components/Storage/Tools/hwconfig/config.py +++ b/Sming/Components/Storage/Tools/hwconfig/config.py @@ -77,8 +77,9 @@ def to_json(self): def buildVars(self): dict = {} + dict['SMING_ARCH_HW'] = self.arch dict['PARTITION_TABLE_OFFSET'] = self.partitions.offset_str() - + dict['SPIFLASH_PARTITION_NAMES'] = " ".join(p.name for p in filter(lambda p: p.device == self.devices[0], self.partitions)) dict['HWCONFIG_DEPENDS'] = " ".join(self.depends) dict.update(self.devices.buildVars()) dict.update(self.partitions.buildVars()) diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index ac421a26e8..16fcb38997 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -45,23 +45,13 @@ define HwExpr $(shell $(HWEXPR) "$1") endef -# Obtain partition information -# $1 -> Partition name -# $2 -> Expression -define PartInfo -$(shell $(HWCONFIG_TOOL) --part $1 expr $(HWCONFIG) - $2) -endef - - # Import PARTITION_TABLE_OFFSET from hardware configuration DEBUG_VARS += PARTITION_TABLE_OFFSET -SMING_ARCH_HW := $(call HwExpr,config.arch) ifeq ($(SMING_ARCH_HW),) $(error Hardware configuration error) else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) $(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) endif -PARTITION_TABLE_OFFSET := $(call HwExpr,(config.partitions.offset_str())) COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) @@ -91,13 +81,12 @@ hwconfig-list: ##List available hardware configurations @echo "Available configurations: $(foreach c,$(ALL_HWCONFIG),\n $(if $(subst $c,,$(HWCONFIG)), ,*) $(shell printf "%-25s" "$c") $(filter %/$c.hw,$(ALL_HWCONFIG_PATHS)))" @echo -# @echo "Available configurations: $(foreach c,$(ALL_HWCONFIG_PATHS),\n $(if $(subst $c,,$(HWCONFIG)), ,*) $c)" - .PHONY: hwconfig-validate hwconfig-validate: $(HWCONFIG_PATH) ##Validate current hardware configuration @echo "Validating hardware config '$(HWCONFIG)'" $(Q) $(HWCONFIG_TOOL) validate $(HWCONFIG) - $(PARTITION_PATH)/schema.json + ##@Building # The partition table @@ -118,7 +107,7 @@ $$(PTARG): CUSTOM_TARGETS += $$(PTARG) endef -# Create build targets for all partitions with 'make' entry +# Create build targets for all partitions with 'build' property DEBUG_VARS += PARTITIONS_WITH_TARGETS PARTITIONS_WITH_TARGETS := $(call HwExpr,(' '.join([part.name for part in filter(lambda part: part.build is not None, config.partitions)]))) @@ -131,23 +120,30 @@ endef ##@Flashing -# $1 -> Filter expression -define PartChunks -$(call HwExpr,(' '.join(['%s=%s' % (part.address_str(), part.filename) for part in filter(lambda part: $1 and part.device.name == 'spiFlash' and part.filename != '', config.partitions)]))) +# $1 -> Partition name +define PartChunk +$(if $(PARTITION_$1_FILENAME),$(PARTITION_$1_ADDRESS)=$(PARTITION_$1_FILENAME)) +endef + +# $1 -> Partition name +define PartRegion +$(PARTITION_$1_ADDRESS),$(PARTITION_$1_SIZE) endef # One flash sector of 0xFF BLANK_BIN := $(PARTITION_PATH)/blank.bin -DEBUG_VARS += FLASH_APP_CHUNKS FLASH_MAP_CHUNKS FLASH_PARTITION_CHUNKS -$(eval FLASH_APP_CHUNKS = $(call PartChunks,(part.type_is('app')))) +# Just the application chunks +FLASH_APP_CHUNKS = $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(if $(filter app,$(PARTITION_$p_TYPE)),$(call PartChunk,$p))) +# Partition map chunk FLASH_MAP_CHUNK := $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) -$(eval FLASH_PARTITION_CHUNKS = $(patsubst %=,,$(call PartChunks,True))) +# All partitions with image files +FLASH_PARTITION_CHUNKS = $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(call PartChunk,$p)) ifdef PART DEBUG_VARS += FLASH_PART_CHUNKS FLASH_PART_REGION -$(eval FLASH_PART_CHUNKS := $(call PartChunks,(part=='$(PART)'))) -$(eval FLASH_PART_REGION := $(call HwExpr,('%s,%s' % (part.address_str(), part.size)))) +FLASH_PART_CHUNKS := $(call PartChunk,$(PART)) +FLASH_PART_REGION := $(call PartRegion,$(PART)) endif # Where to store read partition map file diff --git a/Sming/component.mk b/Sming/component.mk index c0fae08108..e51915e17f 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -159,7 +159,7 @@ flashboot: $(FLASH_BOOT_LOADER) kill_term ##Write just the Bootloader $(call WriteFlash,$(FLASH_BOOT_CHUNKS)) .PHONY: flashapp -flashapp: all kill_term ##Write just the application image +flashapp: all kill_term ##Write just the application image(s) $(call WriteFlash,$(FLASH_APP_CHUNKS)) .PHONY: flash From f50479b1933d10c3e6254ba884cb85c9997b80a3 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 5 Feb 2021 16:58:30 +0000 Subject: [PATCH 23/36] Remove `DISABLE_SPIFFS` --- Sming/Components/spiffs/README.rst | 41 ++++++++++++------- Sming/Components/spiffs/component.mk | 9 ++-- .../CS5460/samples/generic/component.mk | 1 - .../ModbusMaster/samples/generic/component.mk | 1 - .../modbusino/samples/generic/component.mk | 1 - Sming/component.mk | 1 - samples/Accelerometer_MMA7455/component.mk | 2 - samples/Basic_APA102/component.mk | 2 - samples/Basic_AWS/component.mk | 1 - samples/Basic_Audio/component.mk | 2 - samples/Basic_Blink/component.mk | 5 --- samples/Basic_Capsense/component.mk | 2 - samples/Basic_DateTime/component.mk | 2 - samples/Basic_Delegates/component.mk | 1 - samples/Basic_HwPWM/component.mk | 2 - samples/Basic_Interrupts/component.mk | 1 - samples/Basic_NFC/component.mk | 2 - samples/Basic_Neopixel/component.mk | 2 - samples/Basic_ProgMem/component.mk | 1 - samples/Basic_ScannerI2C/component.mk | 1 - samples/Basic_Servo/component.mk | 2 - samples/Basic_SmartConfig/component.mk | 1 - samples/Basic_Ssl/component.mk | 2 - samples/Basic_Tasks/component.mk | 1 - samples/Basic_Utility/component.mk | 3 -- samples/Basic_WebSkeletonApp/README.rst | 2 +- .../Basic_WebSkeletonApp/app/application.cpp | 2 + .../Basic_WebSkeletonApp/app/webserver.cpp | 2 +- samples/Basic_WebSkeletonApp/component.mk | 7 ++++ samples/Basic_WiFi/component.mk | 1 - samples/Basic_rBoot/README.rst | 4 +- samples/CanBus/component.mk | 2 - samples/Compass_HMC5883L/component.mk | 2 - samples/DFPlayerMini/component.mk | 2 - samples/DS3232RTC_NTP_Setter/component.mk | 2 - samples/Display_TM1637/component.mk | 2 - samples/Distance_Vl53l0x/component.mk | 2 - samples/DnsCaptivePortal/component.mk | 1 - samples/FtpServer_Files/component.mk | 2 +- samples/Gesture_APDS-9960/component.mk | 2 - samples/HttpClient_Instapush/component.mk | 2 +- samples/HttpClient_ThingSpeak/component.mk | 2 +- samples/HttpServer_Bootstrap/component.mk | 2 +- samples/Humidity_AM2321/component.mk | 2 - samples/Humidity_DHT22/component.mk | 2 - samples/Humidity_SI7021/component.mk | 2 - samples/IR_lib/component.mk | 2 - samples/LED_WS2812/component.mk | 2 - samples/LED_YeelightBulb/component.mk | 1 - samples/Light_BH1750/component.mk | 2 - samples/LiquidCrystal_44780/component.mk | 2 - samples/LiveDebug/component.mk | 2 - samples/Nextion_Button/component.mk | 2 - samples/PortExpander_MCP23017/component.mk | 2 - samples/Pressure_BMP180/component.mk | 2 - samples/Radio_RCSwitch/component.mk | 2 - samples/Radio_nRF24L01/component.mk | 2 - samples/Radio_si4432/component.mk | 2 - samples/SDCard/component.mk | 1 - samples/ScreenLCD_5110/component.mk | 2 - samples/ScreenOLED_SSD1306/component.mk | 2 - samples/ScreenTFT_ILI9163C/component.mk | 2 - samples/SystemClock_NTP/component.mk | 2 - samples/TcpClient_NarodMon/component.mk | 1 - .../Telnet_TCPServer_TCPClient/component.mk | 1 - samples/Temperature_DS1820/component.mk | 2 - samples/UdpServer_Echo/component.mk | 1 - samples/UdpServer_mDNS/component.mk | 2 +- samples/Ultrasonic_HCSR04/component.mk | 2 - samples/Wifi_Sniffer/component.mk | 1 - tests/SharedComponent/Project/component.mk | 5 --- 71 files changed, 47 insertions(+), 136 deletions(-) delete mode 100644 samples/Basic_Interrupts/component.mk delete mode 100644 samples/Basic_ProgMem/component.mk delete mode 100644 samples/Basic_WiFi/component.mk delete mode 100644 samples/DnsCaptivePortal/component.mk delete mode 100644 samples/TcpClient_NarodMon/component.mk delete mode 100644 samples/Telnet_TCPServer_TCPClient/component.mk delete mode 100644 samples/UdpServer_Echo/component.mk delete mode 100644 samples/Wifi_Sniffer/component.mk diff --git a/Sming/Components/spiffs/README.rst b/Sming/Components/spiffs/README.rst index cefeace69b..13e5852abc 100644 --- a/Sming/Components/spiffs/README.rst +++ b/Sming/Components/spiffs/README.rst @@ -3,34 +3,45 @@ SPIFFS for Sming This Component provides SPIFFS filesystem support for all architectures. -.. envvar:: DISABLE_SPIFFS +A single SPIFFS partition is defined using :envvar:`HWCONFIG` ``=spiffs``, which supports these build variables: - 0 (default) - Enable filesystem generation - - 1 - Disable filesystem generation + .. envvar:: DISABLE_SPIFFS - The value is also #defined for application use. + [deprecated and removed] - Note this doesn't actually disable SPIFFS support in the application! + This value is no longer supported. Please remove it from your project's component.mk file. -.. envvar:: SPIFF_SIZE + .. envvar:: SPIFF_SIZE - Size (in bytes) of the SPIFFS area in Flash memory. + [deprecated and removed] + Size (in bytes) of the SPIFFS area in Flash memory. To change this, edit the :ref:`hardware_config`. -.. envvar:: SPIFF_FILES + .. envvar:: SPIFF_FILES - default: ``files`` + .. envvar:: SPIFF_FILES - The SPIFFS image is built using files from this directory. + default: ``files`` + The SPIFFS image is built using files from this directory, which must exist or the build will fail. -.. envvar:: SPIFF_BIN + If you set this to an empty value, then an empty filesystem will be created. + + + .. envvar:: SPIFF_BIN + + Filename to use for the generated SPIFFS filesystem image. The default is ``spiff_rom``. + + + .. envvar:: SPIFF_BIN_OUT + + [read-only] Shows the full path to the generated image file. + + +For more control over the SPIFFS partition you can create your own partition definition in a +custom :ref:`hardware_config`. - Path to the generated SPIFFS filesystem image. .. envvar:: SPIFF_FILEDESC_COUNT diff --git a/Sming/Components/spiffs/component.mk b/Sming/Components/spiffs/component.mk index dc2f497a39..f82bfa9608 100644 --- a/Sming/Components/spiffs/component.mk +++ b/Sming/Components/spiffs/component.mk @@ -14,17 +14,14 @@ $(COMPONENT_RULE)$(SPIFFY): ## Application -# This controls filesystem generation, it doesn't actually disable SPIFFS support in the application -CONFIG_VARS += DISABLE_SPIFFS -DISABLE_SPIFFS ?= 0 -APP_CFLAGS += -DDISABLE_SPIFFS=$(DISABLE_SPIFFS) +ifdef DISABLE_SPIFFS +$(error DISABLE_SPIFFS is no longer supported; please remove this option from your component.mk file) +endif -ifeq ($(DISABLE_SPIFFS),0) CACHE_VARS += SPIFF_FILES SPIFF_BIN SPIFF_FILES ?= files SPIFF_BIN ?= spiff_rom SPIFF_BIN_OUT := $(FW_BASE)/$(SPIFF_BIN).bin -endif COMPONENT_RELINK_VARS += SPIFF_FILEDESC_COUNT SPIFF_FILEDESC_COUNT ?= 7 diff --git a/Sming/Libraries/CS5460/samples/generic/component.mk b/Sming/Libraries/CS5460/samples/generic/component.mk index 51f1520d50..391ca50c73 100644 --- a/Sming/Libraries/CS5460/samples/generic/component.mk +++ b/Sming/Libraries/CS5460/samples/generic/component.mk @@ -1,2 +1 @@ ARDUINO_LIBRARIES := CS5460 -DISABLE_SPIFFS = 1 diff --git a/Sming/Libraries/ModbusMaster/samples/generic/component.mk b/Sming/Libraries/ModbusMaster/samples/generic/component.mk index 6b0689bce9..2797abfd8b 100644 --- a/Sming/Libraries/ModbusMaster/samples/generic/component.mk +++ b/Sming/Libraries/ModbusMaster/samples/generic/component.mk @@ -1,2 +1 @@ ARDUINO_LIBRARIES := ModbusMaster -DISABLE_SPIFFS = 1 diff --git a/Sming/Libraries/modbusino/samples/generic/component.mk b/Sming/Libraries/modbusino/samples/generic/component.mk index 4163e3ffed..068e9f51a6 100644 --- a/Sming/Libraries/modbusino/samples/generic/component.mk +++ b/Sming/Libraries/modbusino/samples/generic/component.mk @@ -1,5 +1,4 @@ ARDUINO_LIBRARIES := modbusino -DISABLE_SPIFFS = 1 CONFIG_VARS += MB_SLAVE_ADDR MB_SLAVE_ADDR ?= 1 diff --git a/Sming/component.mk b/Sming/component.mk index e51915e17f..17feb9bb08 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -150,7 +150,6 @@ GLOBAL_CFLAGS += -DSTRING_OBJECT_SIZE=$(STRING_OBJECT_SIZE) .PHONY: flashinit flashinit: $(ESPTOOL) $(FLASH_INIT_DATA) | $(FW_BASE) ##Erase your device's flash memory $(info Flash init data default and blank data) - $(info DISABLE_SPIFFS = $(DISABLE_SPIFFS)) $(Q) $(EraseFlash) $(call WriteFlash,$(FLASH_INIT_CHUNKS)) diff --git a/samples/Accelerometer_MMA7455/component.mk b/samples/Accelerometer_MMA7455/component.mk index ce1725567a..2d423a2a56 100644 --- a/samples/Accelerometer_MMA7455/component.mk +++ b/samples/Accelerometer_MMA7455/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := MMA_7455 - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_APA102/component.mk b/samples/Basic_APA102/component.mk index 75ebf23720..3c540d9c36 100644 --- a/samples/Basic_APA102/component.mk +++ b/samples/Basic_APA102/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := APA102 - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_AWS/component.mk b/samples/Basic_AWS/component.mk index bcb042181f..6e25c28d7d 100644 --- a/samples/Basic_AWS/component.mk +++ b/samples/Basic_AWS/component.mk @@ -1,4 +1,3 @@ -DISABLE_SPIFFS := 1 ENABLE_SSL := 1 MQTT_NO_COMPAT := 1 ENABLE_CUSTOM_HEAP := 1 diff --git a/samples/Basic_Audio/component.mk b/samples/Basic_Audio/component.mk index 44197b4522..3da06ba826 100644 --- a/samples/Basic_Audio/component.mk +++ b/samples/Basic_Audio/component.mk @@ -1,5 +1,3 @@ -DISABLE_SPIFFS=1 - # Required if compiling with ENABLE_GDB=1, you must connect debug terminal to alternative serial pins GDB_UART_SWAP=1 diff --git a/samples/Basic_Blink/component.mk b/samples/Basic_Blink/component.mk index aeac5ea204..fcff50daf3 100644 --- a/samples/Basic_Blink/component.mk +++ b/samples/Basic_Blink/component.mk @@ -33,11 +33,6 @@ # Default is 'standard' (no spiffs), can also provide your own #HWCONFIG := spiffs -## SPIFFS options (deprecated: use HWCONFIG) -## 0: (default) use `HWCONFIG=standard` as default -## 1: Use `HWCONFIG=spiffs` -DISABLE_SPIFFS := 1 - ## Select source of content for default `spiffs` partition when built # SPIFF_FILES = files diff --git a/samples/Basic_Capsense/component.mk b/samples/Basic_Capsense/component.mk index 4652678d1e..7b80efa885 100644 --- a/samples/Basic_Capsense/component.mk +++ b/samples/Basic_Capsense/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := CapacitiveSensor - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_DateTime/component.mk b/samples/Basic_DateTime/component.mk index 7df8c5b1c2..3596ab71d1 100644 --- a/samples/Basic_DateTime/component.mk +++ b/samples/Basic_DateTime/component.mk @@ -1,4 +1,2 @@ -DISABLE_SPIFFS = 1 - # Emulate UART 0 ENABLE_HOST_UARTID := 0 diff --git a/samples/Basic_Delegates/component.mk b/samples/Basic_Delegates/component.mk index 2e02dba4b6..7e4d03795a 100644 --- a/samples/Basic_Delegates/component.mk +++ b/samples/Basic_Delegates/component.mk @@ -1,2 +1 @@ -DISABLE_SPIFFS = 1 DEBUG_VERBOSE_LEVEL = 3 diff --git a/samples/Basic_HwPWM/component.mk b/samples/Basic_HwPWM/component.mk index 5794c096f3..81bf5015ae 100644 --- a/samples/Basic_HwPWM/component.mk +++ b/samples/Basic_HwPWM/component.mk @@ -1,4 +1,2 @@ -DISABLE_SPIFFS = 1 - # Uncomment the line below if you want to use Espressif's PWM library. #ENABLE_CUSTOM_PWM=0 diff --git a/samples/Basic_Interrupts/component.mk b/samples/Basic_Interrupts/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/Basic_Interrupts/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_NFC/component.mk b/samples/Basic_NFC/component.mk index 6de711f481..61d0cfc5e2 100644 --- a/samples/Basic_NFC/component.mk +++ b/samples/Basic_NFC/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := MFRC522 - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_Neopixel/component.mk b/samples/Basic_Neopixel/component.mk index 7728b3185e..379801d9b5 100644 --- a/samples/Basic_Neopixel/component.mk +++ b/samples/Basic_Neopixel/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Adafruit_NeoPixel - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_ProgMem/component.mk b/samples/Basic_ProgMem/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/Basic_ProgMem/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_ScannerI2C/component.mk b/samples/Basic_ScannerI2C/component.mk index 973dfe71cf..e69de29bb2 100644 --- a/samples/Basic_ScannerI2C/component.mk +++ b/samples/Basic_ScannerI2C/component.mk @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_Servo/component.mk b/samples/Basic_Servo/component.mk index 08990693e8..b3fea3a944 100644 --- a/samples/Basic_Servo/component.mk +++ b/samples/Basic_Servo/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Servo - -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_SmartConfig/component.mk b/samples/Basic_SmartConfig/component.mk index 30a787416e..c24c9ac9d9 100644 --- a/samples/Basic_SmartConfig/component.mk +++ b/samples/Basic_SmartConfig/component.mk @@ -1,2 +1 @@ -DISABLE_SPIFFS = 1 ENABLE_SMART_CONFIG = 1 diff --git a/samples/Basic_Ssl/component.mk b/samples/Basic_Ssl/component.mk index 089769b112..b994d8280f 100644 --- a/samples/Basic_Ssl/component.mk +++ b/samples/Basic_Ssl/component.mk @@ -1,5 +1,3 @@ -DISABLE_SPIFFS = 1 - # ifeq ($(ENABLE_MALLOC_COUNT),1) COMPONENT_DEPENDS += malloc_count diff --git a/samples/Basic_Tasks/component.mk b/samples/Basic_Tasks/component.mk index 6599e8dede..ba03b6dbd5 100644 --- a/samples/Basic_Tasks/component.mk +++ b/samples/Basic_Tasks/component.mk @@ -1,4 +1,3 @@ -DISABLE_SPIFFS := 1 ARDUINO_LIBRARIES := ArduinoFFT SignalGenerator ENABLE_TASK_COUNT := 1 diff --git a/samples/Basic_Utility/component.mk b/samples/Basic_Utility/component.mk index 2c74f83110..bde909c92d 100644 --- a/samples/Basic_Utility/component.mk +++ b/samples/Basic_Utility/component.mk @@ -5,8 +5,5 @@ APP_NAME := utility # We don't need heap monitoring for utility applications ENABLE_MALLOC_COUNT := 0 -# Don't attempt to build any filesystem images -DISABLE_SPIFFS := 1 - # HOST_NETWORK_OPTIONS := --nonet diff --git a/samples/Basic_WebSkeletonApp/README.rst b/samples/Basic_WebSkeletonApp/README.rst index f6cc42df8b..f46ed941c7 100644 --- a/samples/Basic_WebSkeletonApp/README.rst +++ b/samples/Basic_WebSkeletonApp/README.rst @@ -34,6 +34,6 @@ To test this out, build the application without a filesystem image: .. code-block:: bash - make DISABLE_SPIFFS=1 + make HWCONFIG=standard ENABLE_FLASHSTRING_MAP=1 See *webserver.cpp* for the details. diff --git a/samples/Basic_WebSkeletonApp/app/application.cpp b/samples/Basic_WebSkeletonApp/app/application.cpp index 173a10f07f..187bd579bf 100644 --- a/samples/Basic_WebSkeletonApp/app/application.cpp +++ b/samples/Basic_WebSkeletonApp/app/application.cpp @@ -38,7 +38,9 @@ void init() Serial.systemDebugOutput(true); Serial.commandProcessing(false); +#ifndef ENABLE_FLASHSTRING_MAP spiffs_mount(); // Mount file system, in order to work with files +#endif //SET higher CPU freq & disable wifi sleep // system_update_cpu_freq(SYS_CPU_160MHZ); diff --git a/samples/Basic_WebSkeletonApp/app/webserver.cpp b/samples/Basic_WebSkeletonApp/app/webserver.cpp index 358dfb9884..5e19982205 100644 --- a/samples/Basic_WebSkeletonApp/app/webserver.cpp +++ b/samples/Basic_WebSkeletonApp/app/webserver.cpp @@ -9,7 +9,7 @@ namespace bool serverStarted = false; HttpServer server; -#if DISABLE_SPIFFS +#ifdef ENABLE_FLASHSTRING_MAP // If a filesystem image hasn't been provided, serve the files using a FlashString map #define FILE_LIST(XX) \ diff --git a/samples/Basic_WebSkeletonApp/component.mk b/samples/Basic_WebSkeletonApp/component.mk index c6982bde3d..de15dbd41b 100644 --- a/samples/Basic_WebSkeletonApp/component.mk +++ b/samples/Basic_WebSkeletonApp/component.mk @@ -1,2 +1,9 @@ ARDUINO_LIBRARIES := ArduinoJson6 HWCONFIG := spiffs + +# Use to store files in a FlashString map object instead of SPIFFS +CONFIG_VARS += ENABLE_FLASHSTRING_MAP +ENABLE_FLASHSTRING_MAP ?= 0 +ifeq ($(ENABLE_FLASHSTRING_MAP),1) +COMPONENT_CXXFLAGS += -DENABLE_FLASHSTRING_MAP=1 +endif diff --git a/samples/Basic_WiFi/component.mk b/samples/Basic_WiFi/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/Basic_WiFi/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Basic_rBoot/README.rst b/samples/Basic_rBoot/README.rst index 8b1a836953..2fdcbf7e2d 100644 --- a/samples/Basic_rBoot/README.rst +++ b/samples/Basic_rBoot/README.rst @@ -66,10 +66,10 @@ If you want to use, for example, two 512k roms in the first 1MB block of flash (old style) then Sming will automatically create two separately linked roms. If you are flashing a single rom to multiple 1MB flash blocks, all using the same offset inside their 1MB blocks, only a single rom is created. -See the rBoot readme for further details. +See :component:`rboot` for further details. - If using a very small flash (e.g. 512k) there may be no room for a - spiffs fileystem, disable it with *DISABLE_SPIFFS = 1* + spiffs fileystem, so use *HWCONFIG = standard* - After building copy all the rom*.bin files to the root of your web server. diff --git a/samples/CanBus/component.mk b/samples/CanBus/component.mk index 5db38ef1f8..668373c3fc 100644 --- a/samples/CanBus/component.mk +++ b/samples/CanBus/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := MCP_CAN_lib - -DISABLE_SPIFFS = 1 diff --git a/samples/Compass_HMC5883L/component.mk b/samples/Compass_HMC5883L/component.mk index 763fcb3a77..bfebb2a7dc 100644 --- a/samples/Compass_HMC5883L/component.mk +++ b/samples/Compass_HMC5883L/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := HMC5883L - -DISABLE_SPIFFS = 1 diff --git a/samples/DFPlayerMini/component.mk b/samples/DFPlayerMini/component.mk index ccda75fa45..fe1938596e 100644 --- a/samples/DFPlayerMini/component.mk +++ b/samples/DFPlayerMini/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := DFRobotDFPlayerMini - -DISABLE_SPIFFS = 1 diff --git a/samples/DS3232RTC_NTP_Setter/component.mk b/samples/DS3232RTC_NTP_Setter/component.mk index 4534b60e67..9f593e41f9 100644 --- a/samples/DS3232RTC_NTP_Setter/component.mk +++ b/samples/DS3232RTC_NTP_Setter/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := DS3232RTC - -DISABLE_SPIFFS = 1 diff --git a/samples/Display_TM1637/component.mk b/samples/Display_TM1637/component.mk index 1f36cbd482..76b8ba3655 100644 --- a/samples/Display_TM1637/component.mk +++ b/samples/Display_TM1637/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := TM1637 - -DISABLE_SPIFFS = 1 diff --git a/samples/Distance_Vl53l0x/component.mk b/samples/Distance_Vl53l0x/component.mk index d61fceb68e..a944fc3b10 100644 --- a/samples/Distance_Vl53l0x/component.mk +++ b/samples/Distance_Vl53l0x/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Adafruit_VL53L0X - -DISABLE_SPIFFS = 1 diff --git a/samples/DnsCaptivePortal/component.mk b/samples/DnsCaptivePortal/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/DnsCaptivePortal/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/FtpServer_Files/component.mk b/samples/FtpServer_Files/component.mk index 1014c23996..ab4cb6eed9 100644 --- a/samples/FtpServer_Files/component.mk +++ b/samples/FtpServer_Files/component.mk @@ -1,2 +1,2 @@ HWCONFIG := spiffs -DISABLE_SPIFFS := 1 +SPIFF_FILES := diff --git a/samples/Gesture_APDS-9960/component.mk b/samples/Gesture_APDS-9960/component.mk index 29be2f65a4..6dc722623d 100644 --- a/samples/Gesture_APDS-9960/component.mk +++ b/samples/Gesture_APDS-9960/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := SparkFun_APDS9960 - -DISABLE_SPIFFS = 1 diff --git a/samples/HttpClient_Instapush/component.mk b/samples/HttpClient_Instapush/component.mk index 478f05574c..8786c2333d 100644 --- a/samples/HttpClient_Instapush/component.mk +++ b/samples/HttpClient_Instapush/component.mk @@ -1,3 +1,3 @@ HWCONFIG := spiffs -DISABLE_SPIFFS = 1 +SPIFF_FILES := ARDUINO_LIBRARIES := ArduinoJson6 diff --git a/samples/HttpClient_ThingSpeak/component.mk b/samples/HttpClient_ThingSpeak/component.mk index d4730a5d9f..ab4cb6eed9 100644 --- a/samples/HttpClient_ThingSpeak/component.mk +++ b/samples/HttpClient_ThingSpeak/component.mk @@ -1,2 +1,2 @@ HWCONFIG := spiffs -DISABLE_SPIFFS = 1 +SPIFF_FILES := diff --git a/samples/HttpServer_Bootstrap/component.mk b/samples/HttpServer_Bootstrap/component.mk index 1014c23996..ab4cb6eed9 100644 --- a/samples/HttpServer_Bootstrap/component.mk +++ b/samples/HttpServer_Bootstrap/component.mk @@ -1,2 +1,2 @@ HWCONFIG := spiffs -DISABLE_SPIFFS := 1 +SPIFF_FILES := diff --git a/samples/Humidity_AM2321/component.mk b/samples/Humidity_AM2321/component.mk index ed804579fb..a1e9eff658 100644 --- a/samples/Humidity_AM2321/component.mk +++ b/samples/Humidity_AM2321/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := AM2321 - -DISABLE_SPIFFS = 1 diff --git a/samples/Humidity_DHT22/component.mk b/samples/Humidity_DHT22/component.mk index da47456b52..6a0e948fe4 100644 --- a/samples/Humidity_DHT22/component.mk +++ b/samples/Humidity_DHT22/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := DHTesp - -DISABLE_SPIFFS = 1 diff --git a/samples/Humidity_SI7021/component.mk b/samples/Humidity_SI7021/component.mk index e01c1c0c49..d570b9b3f4 100644 --- a/samples/Humidity_SI7021/component.mk +++ b/samples/Humidity_SI7021/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := SI7021 - -DISABLE_SPIFFS = 1 diff --git a/samples/IR_lib/component.mk b/samples/IR_lib/component.mk index 10e306665b..bf1221327c 100644 --- a/samples/IR_lib/component.mk +++ b/samples/IR_lib/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := IR - -DISABLE_SPIFFS = 1 diff --git a/samples/LED_WS2812/component.mk b/samples/LED_WS2812/component.mk index c3b9fd7313..19ece9de25 100644 --- a/samples/LED_WS2812/component.mk +++ b/samples/LED_WS2812/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := WS2812 - -DISABLE_SPIFFS = 1 diff --git a/samples/LED_YeelightBulb/component.mk b/samples/LED_YeelightBulb/component.mk index 8b99c0071c..68b70f2b6b 100644 --- a/samples/LED_YeelightBulb/component.mk +++ b/samples/LED_YeelightBulb/component.mk @@ -1,2 +1 @@ -DISABLE_SPIFFS = 1 ARDUINO_LIBRARIES := Yeelight diff --git a/samples/Light_BH1750/component.mk b/samples/Light_BH1750/component.mk index 0ff62be148..7f0ae9c735 100644 --- a/samples/Light_BH1750/component.mk +++ b/samples/Light_BH1750/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := BH1750FVI - -DISABLE_SPIFFS = 1 diff --git a/samples/LiquidCrystal_44780/component.mk b/samples/LiquidCrystal_44780/component.mk index 2eaa53c03c..1424c543b2 100644 --- a/samples/LiquidCrystal_44780/component.mk +++ b/samples/LiquidCrystal_44780/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := LiquidCrystal - -DISABLE_SPIFFS = 1 diff --git a/samples/LiveDebug/component.mk b/samples/LiveDebug/component.mk index f82ee2d9d9..e54600cb3c 100644 --- a/samples/LiveDebug/component.mk +++ b/samples/LiveDebug/component.mk @@ -1,5 +1,3 @@ -DISABLE_SPIFFS = 1 - ENABLE_GDB = 1 ENABLE_GDB_CONSOLE ?= 1 diff --git a/samples/Nextion_Button/component.mk b/samples/Nextion_Button/component.mk index c7cc0b302c..538e725470 100644 --- a/samples/Nextion_Button/component.mk +++ b/samples/Nextion_Button/component.mk @@ -1,5 +1,3 @@ ARDUINO_LIBRARIES := ITEADLIB_Arduino_Nextion -DISABLE_SPIFFS = 1 - ENABLE_NEXTION = 1 diff --git a/samples/PortExpander_MCP23017/component.mk b/samples/PortExpander_MCP23017/component.mk index e59988fa90..decdab0b14 100644 --- a/samples/PortExpander_MCP23017/component.mk +++ b/samples/PortExpander_MCP23017/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := MCP23017 - -DISABLE_SPIFFS = 1 diff --git a/samples/Pressure_BMP180/component.mk b/samples/Pressure_BMP180/component.mk index edad90d026..bc6de35d78 100644 --- a/samples/Pressure_BMP180/component.mk +++ b/samples/Pressure_BMP180/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := BMP180 - -DISABLE_SPIFFS = 1 diff --git a/samples/Radio_RCSwitch/component.mk b/samples/Radio_RCSwitch/component.mk index 664aaee609..cd2c1300e9 100644 --- a/samples/Radio_RCSwitch/component.mk +++ b/samples/Radio_RCSwitch/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := RCSwitch - -DISABLE_SPIFFS = 1 diff --git a/samples/Radio_nRF24L01/component.mk b/samples/Radio_nRF24L01/component.mk index 5b82a43543..9a478c319a 100644 --- a/samples/Radio_nRF24L01/component.mk +++ b/samples/Radio_nRF24L01/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := RF24 - -DISABLE_SPIFFS = 1 diff --git a/samples/Radio_si4432/component.mk b/samples/Radio_si4432/component.mk index 2ad241b2cc..fdf6ee7962 100644 --- a/samples/Radio_si4432/component.mk +++ b/samples/Radio_si4432/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := si4432 - -DISABLE_SPIFFS = 1 diff --git a/samples/SDCard/component.mk b/samples/SDCard/component.mk index 0811609ed1..e93d57393d 100644 --- a/samples/SDCard/component.mk +++ b/samples/SDCard/component.mk @@ -1,2 +1 @@ ARDUINO_LIBRARIES := SDCard -DISABLE_SPIFFS := 1 diff --git a/samples/ScreenLCD_5110/component.mk b/samples/ScreenLCD_5110/component.mk index 9e06efca69..2855fb3158 100644 --- a/samples/ScreenLCD_5110/component.mk +++ b/samples/ScreenLCD_5110/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Adafruit_PCD8544 - -DISABLE_SPIFFS = 1 diff --git a/samples/ScreenOLED_SSD1306/component.mk b/samples/ScreenOLED_SSD1306/component.mk index c30fac01ee..72278b67fd 100644 --- a/samples/ScreenOLED_SSD1306/component.mk +++ b/samples/ScreenOLED_SSD1306/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Adafruit_SSD1306 - -DISABLE_SPIFFS = 1 diff --git a/samples/ScreenTFT_ILI9163C/component.mk b/samples/ScreenTFT_ILI9163C/component.mk index 9494698445..a6c5efa5b8 100644 --- a/samples/ScreenTFT_ILI9163C/component.mk +++ b/samples/ScreenTFT_ILI9163C/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := TFT_ILI9163C - -DISABLE_SPIFFS = 1 diff --git a/samples/SystemClock_NTP/component.mk b/samples/SystemClock_NTP/component.mk index 5e0ebcaf7f..2d668a76f1 100644 --- a/samples/SystemClock_NTP/component.mk +++ b/samples/SystemClock_NTP/component.mk @@ -1,5 +1,3 @@ -DISABLE_SPIFFS = 1 - ARDUINO_LIBRARIES = \ Timezone \ SolarCalculator diff --git a/samples/TcpClient_NarodMon/component.mk b/samples/TcpClient_NarodMon/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/TcpClient_NarodMon/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Telnet_TCPServer_TCPClient/component.mk b/samples/Telnet_TCPServer_TCPClient/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/Telnet_TCPServer_TCPClient/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/Temperature_DS1820/component.mk b/samples/Temperature_DS1820/component.mk index 087c1b4647..d5d59373b7 100644 --- a/samples/Temperature_DS1820/component.mk +++ b/samples/Temperature_DS1820/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := DS18S20 - -DISABLE_SPIFFS = 1 diff --git a/samples/UdpServer_Echo/component.mk b/samples/UdpServer_Echo/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/UdpServer_Echo/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/samples/UdpServer_mDNS/component.mk b/samples/UdpServer_mDNS/component.mk index ddb062c800..ed2eaeb314 100644 --- a/samples/UdpServer_mDNS/component.mk +++ b/samples/UdpServer_mDNS/component.mk @@ -1,3 +1,3 @@ HWCONFIG := spiffs -DISABLE_SPIFFS = 1 +SPIFF_FILES := COMPONENT_DEPENDS += mdns diff --git a/samples/Ultrasonic_HCSR04/component.mk b/samples/Ultrasonic_HCSR04/component.mk index c21f2d42fa..793a94abd4 100644 --- a/samples/Ultrasonic_HCSR04/component.mk +++ b/samples/Ultrasonic_HCSR04/component.mk @@ -1,3 +1 @@ ARDUINO_LIBRARIES := Ultrasonic - -DISABLE_SPIFFS = 1 diff --git a/samples/Wifi_Sniffer/component.mk b/samples/Wifi_Sniffer/component.mk deleted file mode 100644 index 973dfe71cf..0000000000 --- a/samples/Wifi_Sniffer/component.mk +++ /dev/null @@ -1 +0,0 @@ -DISABLE_SPIFFS = 1 diff --git a/tests/SharedComponent/Project/component.mk b/tests/SharedComponent/Project/component.mk index 9aaec6713e..80934857ab 100644 --- a/tests/SharedComponent/Project/component.mk +++ b/tests/SharedComponent/Project/component.mk @@ -1,12 +1,7 @@ ## Local build configuration ## Parameters configured here will override default and ENV values. -## SPIFFS options -DISABLE_SPIFFS = 1 -# SPIFF_FILES = files - DEBUG_VERBOSE_LEVEL = 3 -SPI_SIZE = 4M # Tell Sming about any Components we need COMPONENT_DEPENDS := shared-test From 39a175fd351066c46a109fbdebbb506972330e5a Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 5 Feb 2021 17:27:53 +0000 Subject: [PATCH 24/36] Fix SPI_SIZE handling --- Sming/Arch/Esp8266/Components/esp8266/startup.cpp | 2 +- Sming/Arch/Host/Components/vflash/component.mk | 4 ++-- Sming/Components/Storage/README.rst | 4 ++-- Sming/Components/esptool/component.mk | 9 +++++---- Sming/building.rst | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp index 3098cac8db..ef6d1d07d3 100644 --- a/Sming/Arch/Esp8266/Components/esp8266/startup.cpp +++ b/Sming/Arch/Esp8266/Components/esp8266/startup.cpp @@ -92,7 +92,7 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void) os_printf("partition[%u]: %u, 0x%08x, 0x%08x\n", i, part.type, part.addr, part.size); } if(sizeMap < FLASH_SIZE_8M_MAP_512_512) { - os_printf("** Note: SDK 3.0.1 requires SPI_SIZE >= 1M\n"); + os_printf("** Note: SDK 3.0.1 requires spiFlash size >= 1M\n"); } while(1) { // Cannot proceed diff --git a/Sming/Arch/Host/Components/vflash/component.mk b/Sming/Arch/Host/Components/vflash/component.mk index d558aaeda4..98410de609 100644 --- a/Sming/Arch/Host/Components/vflash/component.mk +++ b/Sming/Arch/Host/Components/vflash/component.mk @@ -7,8 +7,8 @@ DD := dd CACHE_VARS += FLASH_BIN FLASH_BIN ?= $(FW_BASE)/flash.bin -CONFIG_VARS += SPI_SIZE -SPI_SIZE ?= 4M +DEBUG_VARS += SPI_SIZE +SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) # Options to add when running emulator CACHE_VARS += HOST_FLASH_OPTIONS diff --git a/Sming/Components/Storage/README.rst b/Sming/Components/Storage/README.rst index 4941c6e5db..e4afc51031 100644 --- a/Sming/Components/Storage/README.rst +++ b/Sming/Components/Storage/README.rst @@ -156,7 +156,7 @@ This is a concise view of your flash partitions. Display it like this:: make map -For the :sample:`Basic_Storage` samnple application, we get this: +For the :sample:`Basic_Storage` sample application, we get this: .. code-block:: text @@ -239,7 +239,7 @@ Configuration Overrides ``standard`` to set 4Mbyte flash size spiffs - Adds a single SPIFFS partition + Adds a single SPIFFS partition. See :component:`spiffs`. Other configurations may be available, depending on architecture. You can see these by running ``make hwconfig-list``. diff --git a/Sming/Components/esptool/component.mk b/Sming/Components/esptool/component.mk index 4004bece99..2a4cf0bbc0 100644 --- a/Sming/Components/esptool/component.mk +++ b/Sming/Components/esptool/component.mk @@ -1,10 +1,11 @@ COMPONENT_LIBNAME := -SPI_SPEED = $(STORAGE_DEVICE_spiFlash_SPEED) -SPI_MODE = $(STORAGE_DEVICE_spiFlash_MODE) -SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) +DEBUG_VARS += SPI_SPEED SPI_MODE SPI_SIZE +SPI_SPEED = $(STORAGE_DEVICE_spiFlash_SPEED) +SPI_MODE = $(STORAGE_DEVICE_spiFlash_MODE) +SPI_SIZE = $(STORAGE_DEVICE_spiFlash_SIZE) -flashimageoptions += -fs $(SPI_SIZE)B -ff $(SPI_SPEED)m -fm $(SPI_MODE) +flashimageoptions += -fs $(SPI_SIZE)B -ff $(SPI_SPEED)m -fm $(SPI_MODE) # Default COM port and speed used for flashing CACHE_VARS += COM_PORT_ESPTOOL COM_SPEED_ESPTOOL diff --git a/Sming/building.rst b/Sming/building.rst index 0588ec57f0..e5c6513b50 100644 --- a/Sming/building.rst +++ b/Sming/building.rst @@ -147,7 +147,7 @@ sessions, and will override any values set in your project’s - Type ``make SPIFF_BIN=test-rom`` to build the project and (if enabled) create a SPIFFS image file called ``test-rom.bin`` -- Type ``make flash COM_PORT=COM4 SPI_MODE=dio SPI_SIZE=4M`` to flash +- Type ``make flash COM_PORT=COM4`` to flash the project and ``test-rom`` SPIFFS image using the provided flash memory settings - Next time you type ``make flash``, the same settings will be used, no From 7fdf16a0e62f750c79746c9844175b2e18eb5ad0 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 5 Feb 2021 20:55:50 +0000 Subject: [PATCH 25/36] Remove DISABLE_SPIFFS, SPI_SIZE and SPIFF_SIZE settings from submodules --- Sming/Components/FlashString | 2 +- .../Libraries/.patches/Arduino_TensorFlowLite/README.rst | 8 -------- Sming/Libraries/Arduino_TensorFlowLite | 2 +- Sming/Libraries/DIAL | 2 +- Sming/Libraries/HueEmulator | 2 +- Sming/Libraries/RapidXML | 2 +- Sming/Libraries/SmingTest | 2 +- Sming/Libraries/TFT_S1D13781 | 2 +- Sming/Libraries/UPnP | 2 +- 9 files changed, 8 insertions(+), 16 deletions(-) delete mode 100644 Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst diff --git a/Sming/Components/FlashString b/Sming/Components/FlashString index 4d0934f499..e0e95dbe53 160000 --- a/Sming/Components/FlashString +++ b/Sming/Components/FlashString @@ -1 +1 @@ -Subproject commit 4d0934f499d9d2a583d1de62c65ce5266f4ec82f +Subproject commit e0e95dbe5399c15a4637eee9dbf46a7cb212ef36 diff --git a/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst b/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst deleted file mode 100644 index 514a20a70a..0000000000 --- a/Sming/Libraries/.patches/Arduino_TensorFlowLite/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -Arduino TensorFlow Lite -======================= - -This library runs TensorFlow machine learning models on microcontrollers, allowing you to build AI/ML applications powered by deep learning and neural networks. - -With the included examples, you can recognize speech, detect people using a camera, and recognise "magic wand" gestures using an accelerometer. - -The examples work best with the Arduino Nano 33 BLE Sense board, which has a microphone and accelerometer. diff --git a/Sming/Libraries/Arduino_TensorFlowLite b/Sming/Libraries/Arduino_TensorFlowLite index 521b889982..9f00dd38f4 160000 --- a/Sming/Libraries/Arduino_TensorFlowLite +++ b/Sming/Libraries/Arduino_TensorFlowLite @@ -1 +1 @@ -Subproject commit 521b8899822ccce6188d7fe1fb246c06aec9ef17 +Subproject commit 9f00dd38f4329b8e884c417aa1bcfbfb15f47913 diff --git a/Sming/Libraries/DIAL b/Sming/Libraries/DIAL index 712c3d4087..4d54bde7a1 160000 --- a/Sming/Libraries/DIAL +++ b/Sming/Libraries/DIAL @@ -1 +1 @@ -Subproject commit 712c3d4087fca7c90336b559edf912e2bcf9b846 +Subproject commit 4d54bde7a1620b85bf53fcb5670f2321ee562835 diff --git a/Sming/Libraries/HueEmulator b/Sming/Libraries/HueEmulator index 20553f6e9e..79d7e79754 160000 --- a/Sming/Libraries/HueEmulator +++ b/Sming/Libraries/HueEmulator @@ -1 +1 @@ -Subproject commit 20553f6e9e223588595e626fc7aa060847df57eb +Subproject commit 79d7e797540519a4fcd9e73a6ca7c6e04715794e diff --git a/Sming/Libraries/RapidXML b/Sming/Libraries/RapidXML index 0b56a775b7..f0dc269f7d 160000 --- a/Sming/Libraries/RapidXML +++ b/Sming/Libraries/RapidXML @@ -1 +1 @@ -Subproject commit 0b56a775b7b3603631734506be4984f094df3685 +Subproject commit f0dc269f7d8bf25d13f36ee92bee3a134e7886d8 diff --git a/Sming/Libraries/SmingTest b/Sming/Libraries/SmingTest index b52ffcd329..36913db2ce 160000 --- a/Sming/Libraries/SmingTest +++ b/Sming/Libraries/SmingTest @@ -1 +1 @@ -Subproject commit b52ffcd3297490a39135d50824374d073c4e384b +Subproject commit 36913db2ce3a5be82abb90d2140988307adfc38d diff --git a/Sming/Libraries/TFT_S1D13781 b/Sming/Libraries/TFT_S1D13781 index e83437794c..4940599366 160000 --- a/Sming/Libraries/TFT_S1D13781 +++ b/Sming/Libraries/TFT_S1D13781 @@ -1 +1 @@ -Subproject commit e83437794c493cb4bb1c486aa5181a227ed2408b +Subproject commit 49405993668fc4dfbd51109e836f3eaa7b0b838f diff --git a/Sming/Libraries/UPnP b/Sming/Libraries/UPnP index 0dc7d07fe8..e04f503c15 160000 --- a/Sming/Libraries/UPnP +++ b/Sming/Libraries/UPnP @@ -1 +1 @@ -Subproject commit 0dc7d07fe8436c96f8c0072acea4d4eca0074139 +Subproject commit e04f503c151f9cd67d7ee06f156ede583b5f06e5 From eab5886d40231bc9eb76fd65bb614212572ac55a Mon Sep 17 00:00:00 2001 From: mikee47 Date: Fri, 5 Feb 2021 22:17:30 +0000 Subject: [PATCH 26/36] Add `component-samples` build target, execute from ESP8266 CI build Tidy up build logic and create a `/out/.built` file to speed up re-testing Bugfixes to RingTone and RapidXML libraries --- Sming/Arch/Esp8266/Tools/ci/build.run.sh | 5 ++++ Sming/Libraries/RapidXML | 2 +- Sming/Libraries/RingTone | 2 +- Sming/Makefile | 29 +++++++++++++++++++++--- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Sming/Arch/Esp8266/Tools/ci/build.run.sh b/Sming/Arch/Esp8266/Tools/ci/build.run.sh index 678b068a60..248021d74b 100755 --- a/Sming/Arch/Esp8266/Tools/ci/build.run.sh +++ b/Sming/Arch/Esp8266/Tools/ci/build.run.sh @@ -5,3 +5,8 @@ $MAKE_PARALLEL samples make clean samples-clean $MAKE_PARALLEL Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3 $MAKE_PARALLEL HttpServer_ConfigNetwork ENABLE_CUSTOM_LWIP=2 STRICT=1 + +# Some samples (UPnP, for example) require more recent compiler +if [ "$BUILD_COMPILER" == "eqt" ]; then + $MAKE_PARALLEL component-samples +fi diff --git a/Sming/Libraries/RapidXML b/Sming/Libraries/RapidXML index f0dc269f7d..cab0af2fda 160000 --- a/Sming/Libraries/RapidXML +++ b/Sming/Libraries/RapidXML @@ -1 +1 @@ -Subproject commit f0dc269f7d8bf25d13f36ee92bee3a134e7886d8 +Subproject commit cab0af2fda5d52a365502750db6e6bd01f585053 diff --git a/Sming/Libraries/RingTone b/Sming/Libraries/RingTone index 65ebef1efe..a62f39ea08 160000 --- a/Sming/Libraries/RingTone +++ b/Sming/Libraries/RingTone @@ -1 +1 @@ -Subproject commit 65ebef1efe71f56ee8d01f158420d4ca07a91fe6 +Subproject commit a62f39ea08867b26c7ace53aedb43769f02cc74f diff --git a/Sming/Makefile b/Sming/Makefile index ee0c49f580..5c1b38a588 100644 --- a/Sming/Makefile +++ b/Sming/Makefile @@ -103,8 +103,9 @@ docs: submodules ##Build the Sming documentation # For integration testing both samples and tests are moved outside of the repo. SMING_PROJECTS_DIR ?= $(abspath $(SMING_HOME)/..) -SAMPLES_DIR := $(call FixPath,$(SMING_PROJECTS_DIR)/samples) -TESTS_DIR := $(call FixPath,$(SMING_PROJECTS_DIR)/tests) +SMING_PROJECTS_DIR := $(call FixPath, $(SMING_PROJECTS_DIR)) +SAMPLES_DIR := $(SMING_PROJECTS_DIR)/samples +TESTS_DIR := $(SMING_PROJECTS_DIR)/tests SAMPLE_NAMES = $(shell ls -1 $(SAMPLES_DIR)) @@ -112,7 +113,23 @@ SAMPLE_NAMES = $(shell ls -1 $(SAMPLES_DIR)) samples: | $(SAMPLE_NAMES) ##Build all sample applications $(SAMPLE_NAMES): - $(Q) $(MAKE) --no-print-directory -C $(SAMPLES_DIR)/$@ + $(Q) $(MAKE) --no-print-directory -C $(SAMPLES_DIR)/$@ PIP_ARGS=-q python-requirements all + +# Build component samples +.PHONY: component-samples +component-samples: submodules ##Build all samples contained in components + $(Q) $(MAKE) --no-print-directory build-component-samples + +# Marks sample as build complete for faster re-testing +BUILT_SUFFIX := /out/.built +COMPONENT_SAMPLE_TARGETS = $(addsuffix $(BUILT_SUFFIX),$(wildcard $(foreach c,$(COMPONENT_SEARCH_DIRS),$c/*/samples/*))) + +.PHONY: build-component-samples +build-component-samples: $(COMPONENT_SAMPLE_TARGETS) + +$(COMPONENT_SAMPLE_TARGETS): + $(Q) $(MAKE) --no-print-directory -C $(@:$(BUILT_SUFFIX)=) PIP_ARGS=-q python-requirements all + $(Q) touch $@ TESTS_COMPLETED = $(addsuffix /.complete,$(call ListSubDirs,$(TESTS_DIR))) @@ -123,6 +140,7 @@ $(TESTS_COMPLETED): $(Q) $(MAKE) -C $(@D) execute $(Q) touch $@ + ##@Cleaning .PHONY: samples-clean @@ -130,6 +148,11 @@ samples-clean: ##Clean all sample applications (all arch/build types) @echo Cleaning all samples... -$(Q) cd $(SAMPLES_DIR) && rm -rf $(addsuffix /out,$(SAMPLE_NAMES)) +.PHONY: component-samples-clean +component-samples-clean: ##Clean all component samples + @echo Cleaning all component samples... + -$(Q) rm -rf $(dir $(COMPONENT_SAMPLE_TARGETS)) + CLEAN_TESTS := $(TESTS_COMPLETED:complete=clean) .PHONY: tests-clean tests-clean: $(CLEAN_TESTS) ##Clean all test applications (all arch/build types) From 11933b146108b8a4cad98c72cc6a76ffba4c658d Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 6 Feb 2021 12:21:57 +0000 Subject: [PATCH 27/36] Add tests from components --- Sming/Makefile | 16 ++++++++++----- tests/HostTests/modules/ArduinoJson6.cpp | 26 +++++++++++++----------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/Sming/Makefile b/Sming/Makefile index 5c1b38a588..815c7d3024 100644 --- a/Sming/Makefile +++ b/Sming/Makefile @@ -22,7 +22,7 @@ all: ##@Cleaning .PHONY: dist-clean -dist-clean: submodules-clean samples-clean docs-clean ##Clean everything for all arch/build types +dist-clean: submodules-clean samples-clean docs-clean tests-clean ##Clean everything for all arch/build types -$(Q) rm -rf out .PHONY: clean @@ -105,7 +105,6 @@ docs: submodules ##Build the Sming documentation SMING_PROJECTS_DIR ?= $(abspath $(SMING_HOME)/..) SMING_PROJECTS_DIR := $(call FixPath, $(SMING_PROJECTS_DIR)) SAMPLES_DIR := $(SMING_PROJECTS_DIR)/samples -TESTS_DIR := $(SMING_PROJECTS_DIR)/tests SAMPLE_NAMES = $(shell ls -1 $(SAMPLES_DIR)) @@ -132,9 +131,16 @@ $(COMPONENT_SAMPLE_TARGETS): $(Q) touch $@ -TESTS_COMPLETED = $(addsuffix /.complete,$(call ListSubDirs,$(TESTS_DIR))) PHONY: tests -tests: $(TESTS_COMPLETED) ##Build and run all test applications +tests: submodules ##Build and run all tests + $(Q) $(MAKE) --no-print-directory build-and-run-tests + +COMPLETED_SUFFIX := /.complete +COMPONENT_TESTS = $(call dirx,$(wildcard $(foreach c,$(COMPONENT_SEARCH_DIRS),$c/*/test/component.mk))) +TESTS_COMPLETED = $(addsuffix $(COMPLETED_SUFFIX),$(wildcard $(SMING_PROJECTS_DIR)/tests/* $(COMPONENT_TESTS))) + +PHONY: build-and-run-tests +build-and-run-tests: $(TESTS_COMPLETED) ##Build and run all test applications $(TESTS_COMPLETED): $(Q) $(MAKE) -C $(@D) execute @@ -161,7 +167,7 @@ tests-clean: $(CLEAN_TESTS) ##Clean all test applications (all arch/build types) .PHONY: $(CLEAN_TESTS) $(CLEAN_TESTS): @echo Cleaning '$(@D)' - $(Q) $(MAKE) -C $(@D) clean + -$(Q) $(MAKE) -C $(@D) clean ##@Tools diff --git a/tests/HostTests/modules/ArduinoJson6.cpp b/tests/HostTests/modules/ArduinoJson6.cpp index 8dbc74d63b..9c3690045b 100644 --- a/tests/HostTests/modules/ArduinoJson6.cpp +++ b/tests/HostTests/modules/ArduinoJson6.cpp @@ -292,15 +292,17 @@ class JsonTest6 : public TestGroup loadStream(fs); } - void speedChecks() + template void check(T func) { -#define CHECK(func) \ - for(unsigned i = 0; i < 4; ++i) { \ - ElapseTimer timer; \ - func; \ - debug_i("Time: %s", timer.elapsedTime().toString().c_str()); \ + for(unsigned i = 0; i < 4; ++i) { + ElapseTimer timer; + func(); + debug_i("Time: %s", timer.elapsedTime().toString().c_str()); + } } + void speedChecks() + { PSTR_ARRAY(filename, "test.json"); if(!fileExist(filename)) { FileStream fs(filename, eFO_CreateNewAlways | eFO_WriteOnly); @@ -312,28 +314,28 @@ class JsonTest6 : public TestGroup TEST_CASE("loadBuffer") { LOAD_FSTR(buffer, Resource::test_json); - CHECK(loadBuffer(buffer, Resource::test_json.length())); + check([&] { loadBuffer(buffer, Resource::test_json.length()); }); } TEST_CASE("loadFlashString") { - CHECK(loadFlashString()); + check([&] { loadFlashString(); }); } TEST_CASE("loadFlashString via Stream (cached)") { - CHECK(loadFlashStringViaStream(false)); + check([&] { loadFlashStringViaStream(false); }); } TEST_CASE("loadFlashString via Stream (un-cached)") { - CHECK(loadFlashStringViaStream(true)); + check([&] { loadFlashStringViaStream(true); }); } TEST_CASE("loadFile") { - CHECK(loadFile(filename)); + check([&] { loadFile(filename); }); } TEST_CASE("loadStream") { FileStream fs(filename); - CHECK(loadStream(fs)); + check([&]() { loadStream(fs); }); } } From 3d63abb0eee446a3fa52c4202aae14b487583e99 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 6 Feb 2021 21:17:15 +0000 Subject: [PATCH 28/36] Remove redundant `python-requirements` calls from CI script --- Sming/Arch/Esp8266/Tools/ci/build.run.sh | 1 - Sming/Arch/Host/Tools/ci/coverity-scan.sh | 2 -- 2 files changed, 3 deletions(-) diff --git a/Sming/Arch/Esp8266/Tools/ci/build.run.sh b/Sming/Arch/Esp8266/Tools/ci/build.run.sh index 248021d74b..9ffe3edc5f 100755 --- a/Sming/Arch/Esp8266/Tools/ci/build.run.sh +++ b/Sming/Arch/Esp8266/Tools/ci/build.run.sh @@ -1,6 +1,5 @@ # Esp8266 build.run.sh -make -C "$SMING_PROJECTS_DIR/samples/HttpServer_FirmwareUpload" python-requirements $MAKE_PARALLEL samples make clean samples-clean $MAKE_PARALLEL Basic_Blink ENABLE_CUSTOM_HEAP=1 DEBUG_VERBOSE_LEVEL=3 diff --git a/Sming/Arch/Host/Tools/ci/coverity-scan.sh b/Sming/Arch/Host/Tools/ci/coverity-scan.sh index d914716e71..f53b2afb6a 100755 --- a/Sming/Arch/Host/Tools/ci/coverity-scan.sh +++ b/Sming/Arch/Host/Tools/ci/coverity-scan.sh @@ -2,8 +2,6 @@ set -e -make -C $SMING_HOME/../samples/HttpServer_FirmwareUpload python-requirements - COVERITY_SCAN_PROJECT_NAME=SmingHub/Sming COVERITY_SCAN_NOTIFICATION_EMAIL="slaff@attachix.com" COVERITY_SCAN_BUILD_COMMAND="$MAKE_PARALLEL Basic_Blink Basic_DateTime Basic_Delegates Basic_Interrupts Basic_ProgMem Basic_Serial Basic_Servo Basic_Ssl HttpServer_FirmwareUpload SMING_ARCH=Host DEBUG_VERBOSE_LEVEL=3" From 8ea9c0cfc7d9e517a058f08e0b35f3333dc80f1a Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 6 Feb 2021 17:28:07 +0000 Subject: [PATCH 29/36] Replace 'flags' with separate `readonly` and `encrypted` properties --- Sming/Arch/Esp32/standard.hw | 7 ++----- Sming/Arch/Esp8266/spiffs-two-roms.hw | 1 - Sming/Arch/Esp8266/standard.hw | 5 +---- Sming/Arch/Host/standard.hw | 1 - .../Storage/Tools/hwconfig/partition.py | 21 +++++++------------ Sming/Components/Storage/schema.json | 7 +++++-- Sming/spiffs.hw | 1 - samples/Basic_Storage/basic_storage.hw | 2 -- samples/Basic_rBoot/basic_rboot.hw | 4 +--- 9 files changed, 16 insertions(+), 33 deletions(-) diff --git a/Sming/Arch/Esp32/standard.hw b/Sming/Arch/Esp32/standard.hw index 4e9931ff28..1f70b3ca4d 100644 --- a/Sming/Arch/Esp32/standard.hw +++ b/Sming/Arch/Esp32/standard.hw @@ -16,22 +16,19 @@ "address": "0x00f000", "size": "0x1000", "type": "data", - "subtype": "phy", - "flags": "" + "subtype": "phy" }, "nvs": { "address": "0x009000", "size": "0x6000", "type": "data", - "subtype": "nvs", - "flags": "" + "subtype": "nvs" }, "factory": { "address": "0x010000", "size": "0x1f0000", "type": "app", "subtype": "factory", - "flags": "", "filename": "$(TARGET_BIN)" } } diff --git a/Sming/Arch/Esp8266/spiffs-two-roms.hw b/Sming/Arch/Esp8266/spiffs-two-roms.hw index 897c826074..9503891e2a 100644 --- a/Sming/Arch/Esp8266/spiffs-two-roms.hw +++ b/Sming/Arch/Esp8266/spiffs-two-roms.hw @@ -7,7 +7,6 @@ "size": "992K", "type": "app", "subtype": "ota_0", - "flags": "", "filename": "$(RBOOT_ROM_1_BIN)" } } diff --git a/Sming/Arch/Esp8266/standard.hw b/Sming/Arch/Esp8266/standard.hw index b2f45be55f..240e56f679 100644 --- a/Sming/Arch/Esp8266/standard.hw +++ b/Sming/Arch/Esp8266/standard.hw @@ -17,22 +17,19 @@ "size": "4K", "type": "data", "subtype": "phy", - "flags": "", "filename": "$(FLASH_INIT_DATA)" }, "sys_param": { "address": "0x004000", "size": "16K", "type": "data", - "subtype": "sysparam", - "flags": "" + "subtype": "sysparam" }, "rom0": { "address": "0x008000", "size": "992K", "type": "app", "subtype": "factory", - "flags": "", "filename": "$(RBOOT_ROM_0_BIN)" } } diff --git a/Sming/Arch/Host/standard.hw b/Sming/Arch/Host/standard.hw index 119ddd8ba8..7572825a4c 100644 --- a/Sming/Arch/Host/standard.hw +++ b/Sming/Arch/Host/standard.hw @@ -14,7 +14,6 @@ "size": "992K", "type": "app", "subtype": "factory", - "flags": "", "filename": "$(BLANK_BIN)" } } diff --git a/Sming/Components/Storage/Tools/hwconfig/partition.py b/Sming/Components/Storage/Tools/hwconfig/partition.py index aa0515e304..e1d01ae19f 100644 --- a/Sming/Components/Storage/Tools/hwconfig/partition.py +++ b/Sming/Components/Storage/Tools/hwconfig/partition.py @@ -314,7 +314,8 @@ def __init__(self, device=None, name="", address=None, size=None, ptype=None, su self.size = size self.type = parse_type(ptype) self.subtype = parse_subtype(self.type, subtype) - self.flags = set() + self.readonly = False + self.encrypted = False self.filename = '' self.build = None @@ -337,12 +338,6 @@ def parse_dict(self, data, devices): self.address = parse_int(v) elif k == 'size': self.size = parse_int(v) - elif k == 'flags': - flags = v.split() - for flag in flags: - if not flag in Entry.FLAGS: - raise InputError("Unknown flag '%s'" % flag) - self.flags |= {flag} elif k == 'filename': self.filename = v elif k == 'build': @@ -377,8 +372,6 @@ def dict(self): res[k] = stringnum(self.type_str()) elif k == 'subtype': res[k] = stringnum(self.subtype_str()) - elif k == 'flags': - res[k] = self.flags_str() elif v is not None and k != 'name': res[k] = v return res @@ -391,7 +384,7 @@ def buildVars(self): dict.pop('build', None) for k, v in dict.items(): k = "PARTITION_%s_%s" % (self.name, k.upper()) - res[k] = v + res[k] = int(v) if type(v) is bool else v return res @@ -422,9 +415,6 @@ def subtype_str(self): def subtype_is(self, subtype): return self.subtype_str() == subtype if isinstance(subtype, str) else self.subtype == subtype - def flags_str(self): - return " ".join(self.flags) - def __eq__(self, other): if isinstance(other, str): return self.name == other @@ -495,7 +485,10 @@ def from_binary(cls, b): return res def to_binary(self): - flags = sum((1 << self.FLAGS[flag]) for flag in self.flags) + flags = 0 + for key in self.FLAGS: + if getattr(self, key): + flags |= (1 << self.FLAGS[key]) return struct.pack(self.STRUCT_FORMAT, self.MAGIC_BYTES, self.type, self.subtype, diff --git a/Sming/Components/Storage/schema.json b/Sming/Components/Storage/schema.json index 8f059cb95e..103b45eb30 100644 --- a/Sming/Components/Storage/schema.json +++ b/Sming/Components/Storage/schema.json @@ -136,8 +136,11 @@ "integer" ] }, - "flags": { - "type": "string" + "readonly": { + "type": "boolean" + }, + "encrypted": { + "type": "boolean" }, "filename": { "type": "string", diff --git a/Sming/spiffs.hw b/Sming/spiffs.hw index 65df953835..f3c0ee4e93 100644 --- a/Sming/spiffs.hw +++ b/Sming/spiffs.hw @@ -7,7 +7,6 @@ "size": "512K", "type": "data", "subtype": "spiffs", - "flags": "", "filename": "$(SPIFF_BIN_OUT)", "build": { "target": "spiffsgen", diff --git a/samples/Basic_Storage/basic_storage.hw b/samples/Basic_Storage/basic_storage.hw index 44ee958b05..c878ea9713 100644 --- a/samples/Basic_Storage/basic_storage.hw +++ b/samples/Basic_Storage/basic_storage.hw @@ -32,7 +32,6 @@ "size": "256K", "type": "data", "subtype": "spiffs", - "flags": "", "filename": "$(FW_BASE)/spiffs1_rom.bin", "build": { "target": "spiffsgen", @@ -44,7 +43,6 @@ "size": "256K", "type": "data", "subtype": "spiffs", - "flags": "", "filename": "$(FW_BASE)/spiffs2_rom.bin", "build": { "target": "spiffsgen", diff --git a/samples/Basic_rBoot/basic_rboot.hw b/samples/Basic_rBoot/basic_rboot.hw index 3f3d1161e7..64465e1b3a 100644 --- a/samples/Basic_rBoot/basic_rboot.hw +++ b/samples/Basic_rBoot/basic_rboot.hw @@ -7,15 +7,13 @@ "size": "992K", "type": "app", "subtype": "ota_0", - "flags": "", "filename": "$(RBOOT_ROM_1_BIN)" }, "spiffs1": { "address": "0x300000", "size": "512K", "type": "data", - "subtype": "spiffs", - "flags": "" + "subtype": "spiffs" } } } From 9e276cc4d1bfa88c8d1940b8803cd5f85f1783b6 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sat, 6 Feb 2021 22:27:36 +0000 Subject: [PATCH 30/36] Fix 'config not found' error during `make dist-clean` --- Sming/Components/Storage/component.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index 16fcb38997..7d5c347a92 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -20,10 +20,12 @@ ALL_HWCONFIG := $(ALL_HWCONFIG:.hw=) HWCONFIG_PATH := $(firstword $(foreach d,$(HWCONFIG_DIRS),$(wildcard $d/$(HWCONFIG).hw))) ifeq (,$(wildcard $(HWCONFIG_PATH))) +ifeq (,$(MAKE_CLEAN)) $(info $(HWCONFIG_DIRS)) $(eval $(call PrintVariable,ALL_HWCONFIG,Available configurations)) $(error Hardware configuration '$(HWCONFIG)' not found) endif +endif PARTITION_PATH := $(COMPONENT_PATH) PARTITION_TOOLS := $(PARTITION_PATH)/Tools From 863e7ab21092d3014e2b05a02cbd0712d7fc88ab Mon Sep 17 00:00:00 2001 From: mikee47 Date: Thu, 28 Jan 2021 15:12:01 +0000 Subject: [PATCH 31/36] Fix network setup for WSL (ifconfig is old, prefer ip) --- Sming/Arch/Host/Tools/setup-network-linux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sming/Arch/Host/Tools/setup-network-linux.sh b/Sming/Arch/Host/Tools/setup-network-linux.sh index 1d2585241a..7f5b7c2df8 100755 --- a/Sming/Arch/Host/Tools/setup-network-linux.sh +++ b/Sming/Arch/Host/Tools/setup-network-linux.sh @@ -17,7 +17,7 @@ fi # Create a tap0 interface with IP network 192.168.13.1/24 sudo ip tuntap add dev tap0 mode tap user $(whoami) sudo ip a a dev tap0 192.168.13.1/24 -sudo ifconfig tap0 up +sudo ip link set tap0 up # The following lines are needed if you plan to access Internet sudo sysctl net.ipv4.ip_forward=1 From 4c4a2004b6875e6bb093dd7ebf59dcee79069ace Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 7 Feb 2021 10:15:33 +0000 Subject: [PATCH 32/36] Include partition table length in build variables --- Sming/Components/Storage/Tools/hwconfig/config.py | 1 + Sming/Components/Storage/Tools/hwconfig/partition.py | 7 ++++--- Sming/Components/Storage/component.mk | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Sming/Components/Storage/Tools/hwconfig/config.py b/Sming/Components/Storage/Tools/hwconfig/config.py index f0f59235c5..b4972375f6 100644 --- a/Sming/Components/Storage/Tools/hwconfig/config.py +++ b/Sming/Components/Storage/Tools/hwconfig/config.py @@ -79,6 +79,7 @@ def buildVars(self): dict['SMING_ARCH_HW'] = self.arch dict['PARTITION_TABLE_OFFSET'] = self.partitions.offset_str() + dict['PARTITION_TABLE_LENGTH'] = "0x%04x" % partition.MAX_PARTITION_LENGTH dict['SPIFLASH_PARTITION_NAMES'] = " ".join(p.name for p in filter(lambda p: p.device == self.devices[0], self.partitions)) dict['HWCONFIG_DEPENDS'] = " ".join(self.depends) dict.update(self.devices.buildVars()) diff --git a/Sming/Components/Storage/Tools/hwconfig/partition.py b/Sming/Components/Storage/Tools/hwconfig/partition.py index e1d01ae19f..1ba64daea9 100644 --- a/Sming/Components/Storage/Tools/hwconfig/partition.py +++ b/Sming/Components/Storage/Tools/hwconfig/partition.py @@ -218,15 +218,16 @@ def verify(self, arch, secure): raise InputError("Partition names must be unique") # check for overlaps + minPartitionAddress = self.offset + PARTITION_TABLE_SIZE dev = '' last = None for p in self: if p.device != dev: last = None dev = p.device - if dev == self[0].device and p.address < self.offset + PARTITION_TABLE_SIZE: - raise InputError("Partition '%s' @ %s-%s is before partition table @ %s" \ - % (p.name, p.address_str(), p.end_str(), addr_format(self.offset + PARTITION_TABLE_SIZE))) + if dev == self[0].device and p.address < minPartitionAddress: + raise InputError("Partition '%s' @ %s-%s must be located after @ %s" \ + % (p.name, p.address_str(), p.end_str(), addr_format(minPartitionAddress))) if last is not None and p.address <= last.end(): raise InputError("Partition '%s' @ %s-%s overlaps '%s' @ %s-%s" \ % (p.name, p.address_str(), p.end_str(), \ diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index 7d5c347a92..6398fca7d6 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -150,7 +150,7 @@ endif # Where to store read partition map file READMAP_BIN := $(OUT_BASE)/partition-table.read.bin -PARTITION_TABLE_REGION := $(PARTITION_TABLE_OFFSET),0x0c00 +PARTITION_TABLE_REGION = $(PARTITION_TABLE_OFFSET),$(PARTITION_TABLE_LENGTH) .PHONY: readmap readmap:##Read partition map from device From a90dda04bb5b3e2c35960bfe87c6f93503cb65a0 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 7 Feb 2021 10:19:08 +0000 Subject: [PATCH 33/36] Fix error message when `readmap` invoked against invalid file --- Sming/Components/Storage/Tools/hwconfig/hwconfig.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py index 527e5c824c..64b2c366b1 100644 --- a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py +++ b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py @@ -43,7 +43,9 @@ def main(): input_is_binary = inputData[0:2] == partition.Entry.MAGIC_BYTES if input_is_binary: config = Config.from_binary(inputData) - if not input_is_binary: + else: + raise InputError("File '%s' not recognised as partition table" % args.input) + else: config = Config.from_name(args.input) partitions = config.partitions From e24c22c2bf3923113218aeff2a499d9b46858469 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 7 Feb 2021 10:19:55 +0000 Subject: [PATCH 34/36] Update schema, `target` is required if partition `build` property defined --- Sming/Components/Storage/schema.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sming/Components/Storage/schema.json b/Sming/Components/Storage/schema.json index 103b45eb30..df887e75e2 100644 --- a/Sming/Components/Storage/schema.json +++ b/Sming/Components/Storage/schema.json @@ -169,7 +169,10 @@ "type": "string", "description": "Makefile target for this build" } - } + }, + "required": [ + "target" + ] } } } \ No newline at end of file From 6b4aeb1675cad5f880d6046fb57f31990ad61638 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 7 Feb 2021 11:46:44 +0000 Subject: [PATCH 35/36] Add sanity check on chunks before writing to flash, critically to ensure image file fits --- .../Storage/Tools/hwconfig/hwconfig.py | 23 ++++++++++-- .../Storage/Tools/hwconfig/partition.py | 9 +++++ Sming/Components/Storage/component.mk | 36 +++++++++++-------- Sming/component.mk | 2 ++ 4 files changed, 54 insertions(+), 16 deletions(-) diff --git a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py index 64b2c366b1..5a8cfbdf9b 100644 --- a/Sming/Components/Storage/Tools/hwconfig/hwconfig.py +++ b/Sming/Components/Storage/Tools/hwconfig/hwconfig.py @@ -27,7 +27,7 @@ def main(): parser.add_argument('--quiet', '-q', help="Don't print non-critical status messages to stderr", action='store_true') parser.add_argument('--secure', help="Require app partitions to be suitable for secure boot", action='store_true') parser.add_argument('--part', help="Name of partition to operate on") - parser.add_argument('command', help='Action to perform', choices=['partgen', 'expr', 'validate']) + parser.add_argument('command', help='Action to perform', choices=['partgen', 'expr', 'validate', 'flashcheck']) parser.add_argument('input', help='Name of hardware configuration or path to binary partition table') parser.add_argument('output', help='Path to output file. Will use stdout if omitted.', nargs='?', default='-') parser.add_argument('expr', help='Expression to evaluate', nargs='?', default=None) @@ -35,7 +35,7 @@ def main(): args = parser.parse_args() common.quiet = args.quiet - + output = None input_is_binary = False if os.path.exists(args.input): @@ -56,6 +56,7 @@ def main(): return if args.command == 'validate': + # Validate resulting hardware configuration against schema try: from jsonschema import Draft7Validator from jsonschema.exceptions import ValidationError @@ -70,12 +71,30 @@ def main(): for e in errors: critical("%s @ %s" % (e.message, e.path)) sys.exit(3) + elif args.command == 'flashcheck': + # Expect list of chunks, such as "0x100000=/out/Esp8266/debug/firmware/spiff_rom.bin 0x200000=custom.bin" + list = args.expr.split() + if len(list) == 0: + raise InputError("No chunks to flash!") + for e in list: + addr, filename = e.split('=') + addr = int(addr, 0) + part = config.partitions.find_by_address(addr) + if part is None: + raise InputError("No partition contains address 0x%08x" % addr) + if part.address != addr: + raise InputError("Address 0x%08x is within partition '%s', not at start (0x%08x)" % (addr, part.name, part.address)) + filesize = os.path.getsize(filename) + if filesize > part.size: + raise InputError("File '%s' is 0x%08x bytes, too big for partition '%s' (0x%08x bytes)" % (os.path.basename(filename), filesize, part.name, part.size)) elif args.command == 'partgen': + # Generate partition table binary if not args.no_verify: status("Verifying partition table...") config.verify(args.secure) output = config.partitions.to_binary(config.devices) elif args.command == 'expr': + # Evaluate expression against configuration data output = str(eval(args.expr)).encode() else: raise InputError('Unknown command: %s' % args.command) diff --git a/Sming/Components/Storage/Tools/hwconfig/partition.py b/Sming/Components/Storage/Tools/hwconfig/partition.py index 1ba64daea9..018167d60a 100644 --- a/Sming/Components/Storage/Tools/hwconfig/partition.py +++ b/Sming/Components/Storage/Tools/hwconfig/partition.py @@ -198,6 +198,12 @@ def find_by_name(self, name): return p return None + def find_by_address(self, addr): + for p in self: + if p.contains(addr): + return p + return None + def verify(self, arch, secure): """Verify partition layout """ @@ -404,6 +410,9 @@ def address_str(self): def size_str(self): return size_format(self.size) + def contains(self, addr): + return addr >= self.address and addr <= self.end() + def type_str(self): return "" if self.type == 0xff else lookup_keyword(self.type, TYPES) diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index 6398fca7d6..ea241d4399 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -41,7 +41,7 @@ HWCONFIG_MK := $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk $(shell $(HWCONFIG_TOOL) --quiet expr $(HWCONFIG) $(HWCONFIG_MK) "config.buildVars()") include $(HWCONFIG_MK) -HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part $(PART)) expr $(HWCONFIG) - +HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part "$(PART)") expr $(HWCONFIG) - define HwExpr $(shell $(HWEXPR) "$1") @@ -122,31 +122,32 @@ endef ##@Flashing -# $1 -> Partition name -define PartChunk -$(if $(PARTITION_$1_FILENAME),$(PARTITION_$1_ADDRESS)=$(PARTITION_$1_FILENAME)) +# Get chunks for a list of partitions, ignore any without filenames +# $1 -> List of partition names +define PartChunks +$(foreach p,$1,$(if $(PARTITION_$p_FILENAME),$(PARTITION_$p_ADDRESS)=$(PARTITION_$p_FILENAME))) endef -# $1 -> Partition name -define PartRegion -$(PARTITION_$1_ADDRESS),$(PARTITION_$1_SIZE) +# Get regions for list of partitions +# $1 -> List of partition names +define PartRegions +$(foreach p,$1,$(PARTITION_$p_ADDRESS),$(PARTITION_$p_SIZE_BYTES)) endef # One flash sector of 0xFF BLANK_BIN := $(PARTITION_PATH)/blank.bin # Just the application chunks -FLASH_APP_CHUNKS = $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(if $(filter app,$(PARTITION_$p_TYPE)),$(call PartChunk,$p))) +SPIFLASH_APP_PARTITION_NAMES := $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(if $(filter app,$(PARTITION_$p_TYPE)),$p)) +FLASH_APP_CHUNKS = $(call PartChunks,$(SPIFLASH_APP_PARTITION_NAMES)) # Partition map chunk FLASH_MAP_CHUNK := $(PARTITION_TABLE_OFFSET)=$(PARTITIONS_BIN) # All partitions with image files -FLASH_PARTITION_CHUNKS = $(foreach p,$(SPIFLASH_PARTITION_NAMES),$(call PartChunk,$p)) +FLASH_PARTITION_CHUNKS = $(call PartChunks,$(SPIFLASH_PARTITION_NAMES)) -ifdef PART -DEBUG_VARS += FLASH_PART_CHUNKS FLASH_PART_REGION -FLASH_PART_CHUNKS := $(call PartChunk,$(PART)) -FLASH_PART_REGION := $(call PartRegion,$(PART)) -endif +# User-selected partition(s) +FLASH_PART_CHUNKS = $(call PartChunks,$(PART)) +FLASH_PART_REGION = $(call PartRegions,$(PART)) # Where to store read partition map file READMAP_BIN := $(OUT_BASE)/partition-table.read.bin @@ -160,8 +161,15 @@ readmap:##Read partition map from device $(Q) $(HWCONFIG_TOOL) expr $(READMAP_BIN) - "config.map().to_csv()" @echo +# Run sanity checks against a list of partitions before flashing commences +# $1 -> List of partition names +define CheckPartitionChunks +$(HWCONFIG_TOOL) flashcheck $(HWCONFIG) - "$1" +endef + .PHONY: flashpart flashpart: all kill_term ##Flash a specific partition, set PART=name + $(Q) $(call CheckPartitionChunks,$(FLASH_PART_CHUNKS)) $(call WriteFlash,$(FLASH_PART_CHUNKS)) .PHONY: erasepart diff --git a/Sming/component.mk b/Sming/component.mk index 17feb9bb08..9685debd3a 100644 --- a/Sming/component.mk +++ b/Sming/component.mk @@ -159,10 +159,12 @@ flashboot: $(FLASH_BOOT_LOADER) kill_term ##Write just the Bootloader .PHONY: flashapp flashapp: all kill_term ##Write just the application image(s) + $(Q) $(call CheckPartitionChunks,$(FLASH_APP_CHUNKS)) $(call WriteFlash,$(FLASH_APP_CHUNKS)) .PHONY: flash flash: all kill_term ##Write the boot loader and all defined partition images + $(Q) $(call CheckPartitionChunks,$(FLASH_PARTITION_CHUNKS)) $(call WriteFlash,$(FLASH_BOOT_CHUNKS) $(FLASH_MAP_CHUNK) $(FLASH_PARTITION_CHUNKS)) ifeq ($(ENABLE_GDB), 1) $(GDB_CMDLINE) From 2e76211f70d4310cdea82d88c3dc881b1f0f8df4 Mon Sep 17 00:00:00 2001 From: mikee47 Date: Sun, 7 Feb 2021 12:05:51 +0000 Subject: [PATCH 36/36] Fix warnings when running `make dist-clean` --- Sming/Components/Storage/component.mk | 18 ++++++++---------- Sming/Makefile | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/Sming/Components/Storage/component.mk b/Sming/Components/Storage/component.mk index ea241d4399..af0307ec78 100644 --- a/Sming/Components/Storage/component.mk +++ b/Sming/Components/Storage/component.mk @@ -34,28 +34,26 @@ HWCONFIG_TOOL := \ BUILD_BASE=$(BUILD_BASE) \ $(PYTHON) $(PARTITION_TOOLS)/hwconfig/hwconfig.py -ifeq (,$(MAKE_DOCS)) +ifeq (,$(MAKE_DOCS)$(MAKE_CLEAN)) # Generate build variables from hardware configuration HWCONFIG_MK := $(PROJECT_DIR)/$(OUT_BASE)/hwconfig.mk $(shell $(HWCONFIG_TOOL) --quiet expr $(HWCONFIG) $(HWCONFIG_MK) "config.buildVars()") include $(HWCONFIG_MK) +ifeq ($(SMING_ARCH_HW),) +$(error Hardware configuration error) +else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) +$(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) +endif +COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) +# Function to evaluate expression against config HWEXPR := $(HWCONFIG_TOOL) $(if $(PART),--part "$(PART)") expr $(HWCONFIG) - define HwExpr $(shell $(HWEXPR) "$1") endef -# Import PARTITION_TABLE_OFFSET from hardware configuration -DEBUG_VARS += PARTITION_TABLE_OFFSET -ifeq ($(SMING_ARCH_HW),) -$(error Hardware configuration error) -else ifneq ($(SMING_ARCH),$(SMING_ARCH_HW)) -$(error Hardware configuration is for arch $(SMING_ARCH_HW), does not match SMING_ARCH ($(SMING_ARCH))) -endif -COMPONENT_CXXFLAGS := -DPARTITION_TABLE_OFFSET=$(PARTITION_TABLE_OFFSET) - ##@Configuration diff --git a/Sming/Makefile b/Sming/Makefile index 815c7d3024..4ad8476a79 100644 --- a/Sming/Makefile +++ b/Sming/Makefile @@ -161,7 +161,7 @@ component-samples-clean: ##Clean all component samples CLEAN_TESTS := $(TESTS_COMPLETED:complete=clean) .PHONY: tests-clean -tests-clean: $(CLEAN_TESTS) ##Clean all test applications (all arch/build types) +tests-clean: $(wildcard $(CLEAN_TESTS)) ##Clean all test applications (all arch/build types) -$(Q) rm -f $(TESTS_COMPLETED) .PHONY: $(CLEAN_TESTS)