From 1694e84f4e15873a43bba215c3b3852273496eaa Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 18:08:51 +0400 Subject: [PATCH 1/8] Resource unpacking core --- .../updater/scenes/updater_scene_main.c | 2 +- applications/updater/util/update_task.c | 3 +- applications/updater/util/update_task.h | 3 +- .../updater/util/update_task_workers.c | 77 ++++++++++++++----- lib/update_util/update_manifest.c | 3 + lib/update_util/update_manifest.h | 1 + 6 files changed, 67 insertions(+), 22 deletions(-) diff --git a/applications/updater/scenes/updater_scene_main.c b/applications/updater/scenes/updater_scene_main.c index d7b28a9db5d..31a212e7341 100644 --- a/applications/updater/scenes/updater_scene_main.c +++ b/applications/updater/scenes/updater_scene_main.c @@ -73,7 +73,7 @@ bool updater_scene_main_on_event(void* context, SceneManagerEvent event) { case UpdaterCustomEventRetryUpdate: if(!update_task_is_running(updater->update_task) && - (update_task_get_state(updater->update_task)->stage != UpdateTaskStageComplete)) + (update_task_get_state(updater->update_task)->stage != UpdateTaskStageCompleted)) update_task_start(updater->update_task); consumed = true; break; diff --git a/applications/updater/util/update_task.c b/applications/updater/util/update_task.c index 8ca5aad2116..197d42f5d3c 100644 --- a/applications/updater/util/update_task.c +++ b/applications/updater/util/update_task.c @@ -19,7 +19,8 @@ static const char* update_task_stage_descr[] = { [UpdateTaskStageRadioCommit] = "Applying radio stack", [UpdateTaskStageLfsBackup] = "Backing up LFS", [UpdateTaskStageLfsRestore] = "Restoring LFS", - [UpdateTaskStageComplete] = "Complete", + [UpdateTaskStageResourcesUpdate] = "Updating resources", + [UpdateTaskStageCompleted] = "Completed!", [UpdateTaskStageError] = "Error", }; diff --git a/applications/updater/util/update_task.h b/applications/updater/util/update_task.h index 32dea989ca0..2325824cb02 100644 --- a/applications/updater/util/update_task.h +++ b/applications/updater/util/update_task.h @@ -23,7 +23,8 @@ typedef enum { UpdateTaskStageRadioCommit, UpdateTaskStageLfsBackup, UpdateTaskStageLfsRestore, - UpdateTaskStageComplete, + UpdateTaskStageResourcesUpdate, + UpdateTaskStageCompleted, UpdateTaskStageError, } UpdateTaskStage; diff --git a/applications/updater/util/update_task_workers.c b/applications/updater/util/update_task_workers.c index fb1e86becbe..e371d9869c3 100644 --- a/applications/updater/util/update_task_workers.c +++ b/applications/updater/util/update_task_workers.c @@ -8,6 +8,7 @@ #include #include #include +#include #define CHECK_RESULT(x) \ if(!(x)) { \ @@ -19,6 +20,8 @@ /* Written into DFU file by build pipeline */ #define FLIPPER_ZERO_DFU_DEVICE_CODE 0xFFFF +#define EXT_PATH "/ext" + static const DfuValidationParams flipper_dfu_params = { .device = FLIPPER_ZERO_DFU_DEVICE_CODE, .product = STM_DFU_PRODUCT_ID, @@ -85,7 +88,7 @@ int32_t update_task_worker_flash_writer(void* context) { CHECK_RESULT(dfu_file_process_targets(&page_task, update_task->file, valid_targets)); } - update_task_set_progress(update_task, UpdateTaskStageComplete, 100); + update_task_set_progress(update_task, UpdateTaskStageCompleted, 100); furi_hal_rtc_set_boot_mode(FuriHalRtcBootModePostUpdate); @@ -99,6 +102,57 @@ int32_t update_task_worker_flash_writer(void* context) { return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED; } +static bool update_task_pre_update(UpdateTask* update_task) { + bool success = false; + string_t backup_file_path; + string_init(backup_file_path); + path_concat( + string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path); + + update_task->state.total_stages = 1; + update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0); + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops + if((success = lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) { + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate); + } + + string_clear(backup_file_path); + return success; +} + +static bool update_task_post_update(UpdateTask* update_task) { + bool success = false; + string_t file_path; + string_init(file_path); + path_concat(string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path); + + update_task->state.total_stages = 1; + bool unpack_resouces = !string_empty_p(update_task->manifest->resource_bundle); + if(unpack_resouces) { + update_task->state.total_stages++; + } + update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); + success = lfs_backup_unpack(update_task->storage, string_get_cstr(file_path)); + if((success = (success && unpack_resouces))) { + update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0); + + path_concat( + string_get_cstr(update_task->update_path), + string_get_cstr(update_task->manifest->resource_bundle), + file_path); + + TarArchive* archive = tar_archive_alloc(update_task->storage); + update_task_set_progress(update_task, UpdateTaskStageProgress, 10); + success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ) && + tar_archive_unpack_to(archive, EXT_PATH); + tar_archive_free(archive); + } + + string_clear(file_path); + return success; +} + int32_t update_task_worker_backup_restore(void* context) { furi_assert(context); UpdateTask* update_task = context; @@ -112,37 +166,22 @@ int32_t update_task_worker_backup_restore(void* context) { } update_task->state.current_stage_idx = 0; - update_task->state.total_stages = 1; if(!update_operation_get_current_package_path(update_task->storage, update_task->update_path)) { return UPDATE_TASK_FAILED; } - string_t backup_file_path; - string_init(backup_file_path); - path_concat( - string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path); - if(boot_mode == FuriHalRtcBootModePreUpdate) { - update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0); - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops - if((success = - lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) { - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate); - } + success = update_task_pre_update(update_task); } else if(boot_mode == FuriHalRtcBootModePostUpdate) { - update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); - success = lfs_backup_unpack(update_task->storage, string_get_cstr(backup_file_path)); + success = update_task_post_update(update_task); } if(success) { - update_task_set_progress(update_task, UpdateTaskStageComplete, 100); + update_task_set_progress(update_task, UpdateTaskStageCompleted, 100); } else { update_task_set_progress(update_task, UpdateTaskStageError, update_task->state.progress); } - string_clear(backup_file_path); - return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED; } diff --git a/lib/update_util/update_manifest.c b/lib/update_util/update_manifest.c index 2ae52f17746..b94ec5784c2 100644 --- a/lib/update_util/update_manifest.c +++ b/lib/update_util/update_manifest.c @@ -10,6 +10,7 @@ UpdateManifest* update_manifest_alloc() { string_init(update_manifest->firmware_dfu_image); string_init(update_manifest->radio_image); string_init(update_manifest->staged_loader_file); + string_init(update_manifest->resource_bundle); update_manifest->target = 0; update_manifest->valid = false; return update_manifest; @@ -21,6 +22,7 @@ void update_manifest_free(UpdateManifest* update_manifest) { string_clear(update_manifest->firmware_dfu_image); string_clear(update_manifest->radio_image); string_clear(update_manifest->staged_loader_file); + string_clear(update_manifest->resource_bundle); free(update_manifest); } @@ -51,6 +53,7 @@ static bool flipper_format_read_string(flipper_file, "Radio", update_manifest->radio_image); flipper_format_read_hex( flipper_file, "Radio address", (uint8_t*)&update_manifest->radio_address, sizeof(uint32_t)); + flipper_format_read_string(flipper_file, "Resources", update_manifest->resource_bundle); return update_manifest->valid; } diff --git a/lib/update_util/update_manifest.h b/lib/update_util/update_manifest.h index 7b60f58a24b..f5a70917c03 100644 --- a/lib/update_util/update_manifest.h +++ b/lib/update_util/update_manifest.h @@ -21,6 +21,7 @@ typedef struct { string_t firmware_dfu_image; string_t radio_image; uint32_t radio_address; + string_t resource_bundle; bool valid; } UpdateManifest; From 4fb7e0c8959d8e10f277078b8bb7f6990e19ff99 Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 19:48:33 +0400 Subject: [PATCH 2/8] Added more fields to manifest; updated dist scripts --- Makefile | 7 +++- lib/toolbox/tar/tar_archive.c | 4 +++ lib/update_util/update_manifest.c | 58 +++++++++++++++++++++++++------ lib/update_util/update_manifest.h | 2 ++ scripts/dist.py | 3 ++ scripts/update.py | 37 +++++++++++++++++--- 6 files changed, 94 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 02008033dc3..59f5a4def9c 100644 --- a/Makefile +++ b/Makefile @@ -88,10 +88,15 @@ updater_clean: updater_debug: @$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug -.PHONY: updater_package +.PHONY: updater_package_bin updater_package: firmware_all updater @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)" +.PHONY: updater_package +updater_package: firmware_all updater + @$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources + @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)" + .PHONY: flash_radio flash_radio: @$(PROJECT_ROOT)/scripts/flash.py core2radio 0x080D7000 $(COPRO_DIR)/stm32wb5x_BLE_Stack_light_fw.bin diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index 7be68bd8f64..c85bfeea61d 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -144,6 +144,10 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, TarArchiveDirectoryOpParams* op_params = param; string_t fname; + if(strcmp(header->name, ".") == 0) { + return 0; + } + if(header->type == MTAR_TDIR) { string_init(fname); path_concat(op_params->work_dir, header->name, fname); diff --git a/lib/update_util/update_manifest.c b/lib/update_util/update_manifest.c index b94ec5784c2..5ec942c897c 100644 --- a/lib/update_util/update_manifest.c +++ b/lib/update_util/update_manifest.c @@ -4,6 +4,17 @@ #include #include +#define MANIFEST_KEY_INFO "Info" +#define MANIFEST_KEY_TARGET "Target" +#define MANIFEST_KEY_LOADER_FILE "Loader" +#define MANIFEST_KEY_LOADER_CRC "Loader CRC" +#define MANIFEST_KEY_DFU_FILE "Firmware" +#define MANIFEST_KEY_RADIO_FILE "Radio" +#define MANIFEST_KEY_RADIO_ADDRESS "Radio address" +#define MANIFEST_KEY_RADIO_VERSION "Radio version" +#define MANIFEST_KEY_RADIO_CRC "Radio CRC" +#define MANIFEST_KEY_ASSETS_FILE "Assets" + UpdateManifest* update_manifest_alloc() { UpdateManifest* update_manifest = malloc(sizeof(UpdateManifest)); string_init(update_manifest->version); @@ -38,22 +49,47 @@ static bool string_init(filetype); update_manifest->valid = flipper_format_read_header(flipper_file, filetype, &version) && - flipper_format_read_string(flipper_file, "Info", update_manifest->version) && - flipper_format_read_uint32(flipper_file, "Target", &update_manifest->target, 1) && - flipper_format_read_string(flipper_file, "Loader", update_manifest->staged_loader_file) && + flipper_format_read_string(flipper_file, MANIFEST_KEY_INFO, update_manifest->version) && + flipper_format_read_uint32( + flipper_file, MANIFEST_KEY_TARGET, &update_manifest->target, 1) && + flipper_format_read_string( + flipper_file, MANIFEST_KEY_LOADER_FILE, update_manifest->staged_loader_file) && flipper_format_read_hex( flipper_file, - "Loader CRC", + MANIFEST_KEY_LOADER_CRC, (uint8_t*)&update_manifest->staged_loader_crc, sizeof(uint32_t)); string_clear(filetype); - /* Optional fields - we can have dfu, radio, or both */ - flipper_format_read_string(flipper_file, "Firmware", update_manifest->firmware_dfu_image); - flipper_format_read_string(flipper_file, "Radio", update_manifest->radio_image); - flipper_format_read_hex( - flipper_file, "Radio address", (uint8_t*)&update_manifest->radio_address, sizeof(uint32_t)); - flipper_format_read_string(flipper_file, "Resources", update_manifest->resource_bundle); + if(update_manifest->valid) { + /* Optional fields - we can have dfu, radio, or both */ + flipper_format_read_string( + flipper_file, MANIFEST_KEY_DFU_FILE, update_manifest->firmware_dfu_image); + flipper_format_read_string( + flipper_file, MANIFEST_KEY_RADIO_FILE, update_manifest->radio_image); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_ADDRESS, + (uint8_t*)&update_manifest->radio_address, + sizeof(uint32_t)); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_VERSION, + (uint8_t*)&update_manifest->radio_version, + sizeof(uint32_t)); + flipper_format_read_hex( + flipper_file, + MANIFEST_KEY_RADIO_CRC, + (uint8_t*)&update_manifest->radio_crc, + sizeof(uint32_t)); + flipper_format_read_string( + flipper_file, MANIFEST_KEY_ASSETS_FILE, update_manifest->resource_bundle); + + update_manifest->valid = + (!string_empty_p(update_manifest->firmware_dfu_image) || + !string_empty_p(update_manifest->radio_image) || + !string_empty_p(update_manifest->resource_bundle)); + } return update_manifest->valid; } @@ -86,4 +122,4 @@ bool update_manifest_init_mem( flipper_format_free(flipper_file); return update_manifest->valid; -} +} \ No newline at end of file diff --git a/lib/update_util/update_manifest.h b/lib/update_util/update_manifest.h index f5a70917c03..7d4757c4b5c 100644 --- a/lib/update_util/update_manifest.h +++ b/lib/update_util/update_manifest.h @@ -21,6 +21,8 @@ typedef struct { string_t firmware_dfu_image; string_t radio_image; uint32_t radio_address; + uint32_t radio_version; + uint32_t radio_crc; string_t resource_bundle; bool valid; } UpdateManifest; diff --git a/scripts/dist.py b/scripts/dist.py index 172ecc01be7..8039e2ffb63 100755 --- a/scripts/dist.py +++ b/scripts/dist.py @@ -18,6 +18,7 @@ def init(self): self.parser_copy.add_argument("-t", dest="target", required=True) self.parser_copy.add_argument("-p", dest="projects", nargs="+", required=True) self.parser_copy.add_argument("-s", dest="suffix", required=True) + self.parser_copy.add_argument("-a", dest="assets", required=False) self.parser_copy.add_argument( "--bundlever", dest="version", @@ -83,6 +84,8 @@ def copy(self): "-stage", self.get_dist_filepath(self.get_project_filename("updater", "bin")), ] + if self.args.assets: + bundle_args.extend(("-a", self.args.assets,)) self.logger.info( f"Use this directory to self-update your Flipper:\n\t{bundle_dir}" ) diff --git a/scripts/update.py b/scripts/update.py index 7e49aad09ad..28ce4de337e 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -3,12 +3,16 @@ from flipper.app import App from flipper.utils.fff import FlipperFormatFile from os.path import basename, join, exists -from os import makedirs +import os import shutil import zlib +import tarfile class Main(App): + # No compression, plain tar + ASSET_TAR_MODE = "w:" + def init(self): self.subparsers = self.parser.add_subparsers(help="sub-command help") @@ -20,24 +24,37 @@ def init(self): self.parser_generate.add_argument("-d", dest="directory", required=True) self.parser_generate.add_argument("-v", dest="version", required=True) self.parser_generate.add_argument("-t", dest="target", required=True) - self.parser_generate.add_argument("-dfu", dest="dfu", required=True) + self.parser_generate.add_argument("-dfu", dest="dfu", required=False) + self.parser_generate.add_argument("-a", dest="assets", required=False) self.parser_generate.add_argument("-stage", dest="stage", required=True) - self.parser_generate.add_argument("-radio", dest="radiobin", required=False) + self.parser_generate.add_argument( + "-radio", dest="radiobin", default="", required=False + ) self.parser_generate.add_argument( "-radioaddr", dest="radioaddr", required=False ) + self.parser_generate.add_argument( + "-radiover", dest="radioversion", required=False + ) self.parser_generate.set_defaults(func=self.generate) def generate(self): stage_basename = basename(self.args.stage) dfu_basename = basename(self.args.dfu) + radiobin_basename = basename(self.args.radiobin) + assets_basename = "" if not exists(self.args.directory): - makedirs(self.args.directory) + os.makedirs(self.args.directory) shutil.copyfile(self.args.stage, join(self.args.directory, stage_basename)) shutil.copyfile(self.args.dfu, join(self.args.directory, dfu_basename)) + if radiobin_basename: + shutil.copyfile(self.args.radiobin, join(self.args.directory, radiobin_basename)) + if self.args.assets: + assets_basename = "assets.tar" + self.package_assets(self.args.assets, join(self.args.directory, assets_basename)) file = FlipperFormatFile() file.setHeader("Flipper firmware upgrade configuration", 1) @@ -47,12 +64,22 @@ def generate(self): file.writeComment("little-endian hex!") file.writeKey("Loader CRC", self.int2ffhex(self.crc(self.args.stage))) file.writeKey("Firmware", dfu_basename) - file.writeKey("Radio", self.args.radiobin or "") + file.writeKey("Radio", radiobin_basename or "") file.writeKey("Radio address", self.int2ffhex(self.args.radioaddr or 0)) + file.writeKey("Radio version", self.int2ffhex(self.args.radioversion or 0)) + if radiobin_basename: + file.writeKey("Radio CRC", self.int2ffhex(self.crc(self.args.radiobin))) + else: + file.writeKey("Radio CRC", self.int2ffhex(0)) + file.writeKey("Assets", assets_basename) file.save("%s/update.fuf" % self.args.directory) return 0 + def package_assets(self, srcdir: str, dst_name: str): + with tarfile.open(dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT) as tarball: + tarball.add(srcdir, arcname="") + @staticmethod def int2ffhex(value: int): hexstr = "%08X" % value From 852af8781c0274eb4787fa0df425af2223862658 Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 19:53:47 +0400 Subject: [PATCH 3/8] Python linter fixes --- scripts/dist.py | 7 ++++++- scripts/update.py | 12 +++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/scripts/dist.py b/scripts/dist.py index 8039e2ffb63..541c386eee4 100755 --- a/scripts/dist.py +++ b/scripts/dist.py @@ -85,7 +85,12 @@ def copy(self): self.get_dist_filepath(self.get_project_filename("updater", "bin")), ] if self.args.assets: - bundle_args.extend(("-a", self.args.assets,)) + bundle_args.extend( + ( + "-a", + self.args.assets, + ) + ) self.logger.info( f"Use this directory to self-update your Flipper:\n\t{bundle_dir}" ) diff --git a/scripts/update.py b/scripts/update.py index 28ce4de337e..83a8d84f7dd 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -51,10 +51,14 @@ def generate(self): shutil.copyfile(self.args.stage, join(self.args.directory, stage_basename)) shutil.copyfile(self.args.dfu, join(self.args.directory, dfu_basename)) if radiobin_basename: - shutil.copyfile(self.args.radiobin, join(self.args.directory, radiobin_basename)) + shutil.copyfile( + self.args.radiobin, join(self.args.directory, radiobin_basename) + ) if self.args.assets: assets_basename = "assets.tar" - self.package_assets(self.args.assets, join(self.args.directory, assets_basename)) + self.package_assets( + self.args.assets, join(self.args.directory, assets_basename) + ) file = FlipperFormatFile() file.setHeader("Flipper firmware upgrade configuration", 1) @@ -77,7 +81,9 @@ def generate(self): return 0 def package_assets(self, srcdir: str, dst_name: str): - with tarfile.open(dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT) as tarball: + with tarfile.open( + dst_name, self.ASSET_TAR_MODE, format=tarfile.USTAR_FORMAT + ) as tarball: tarball.add(srcdir, arcname="") @staticmethod From 83bcfd601e315f8f6f824ca14db6911e80b45e85 Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 21:27:10 +0400 Subject: [PATCH 4/8] Parsing manifest before unpacking --- .../updater/util/update_task_workers.c | 54 +++++++++++-------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/applications/updater/util/update_task_workers.c b/applications/updater/util/update_task_workers.c index e371d9869c3..31f018dcf43 100644 --- a/applications/updater/util/update_task_workers.c +++ b/applications/updater/util/update_task_workers.c @@ -122,32 +122,42 @@ static bool update_task_pre_update(UpdateTask* update_task) { static bool update_task_post_update(UpdateTask* update_task) { bool success = false; + string_t file_path; string_init(file_path); - path_concat(string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path); - update_task->state.total_stages = 1; - bool unpack_resouces = !string_empty_p(update_task->manifest->resource_bundle); - if(unpack_resouces) { - update_task->state.total_stages++; - } - update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); - furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); - success = lfs_backup_unpack(update_task->storage, string_get_cstr(file_path)); - if((success = (success && unpack_resouces))) { - update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0); + update_task->state.total_stages = 2; + do { + CHECK_RESULT(update_task_parse_manifest(update_task)); path_concat( - string_get_cstr(update_task->update_path), - string_get_cstr(update_task->manifest->resource_bundle), - file_path); - - TarArchive* archive = tar_archive_alloc(update_task->storage); - update_task_set_progress(update_task, UpdateTaskStageProgress, 10); - success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ) && - tar_archive_unpack_to(archive, EXT_PATH); - tar_archive_free(archive); - } + string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path); + + bool unpack_resouces = !string_empty_p(update_task->manifest->resource_bundle); + if(unpack_resouces) { + update_task->state.total_stages++; + } + + update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0); + furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); + + CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path))); + + if(unpack_resouces) { + update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0); + + path_concat( + string_get_cstr(update_task->update_path), + string_get_cstr(update_task->manifest->resource_bundle), + file_path); + + TarArchive* archive = tar_archive_alloc(update_task->storage); + update_task_set_progress(update_task, UpdateTaskStageProgress, 10); + success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ) && + tar_archive_unpack_to(archive, EXT_PATH); + tar_archive_free(archive); + } + } while(false); string_clear(file_path); return success; @@ -184,4 +194,4 @@ int32_t update_task_worker_backup_restore(void* context) { } return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED; -} +} \ No newline at end of file From 8c04f5ac8444bc4f97485673253db239c6362dcc Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 21:48:52 +0400 Subject: [PATCH 5/8] Updated pipelines for separate resource build --- .github/workflows/build.yml | 4 +--- Makefile | 9 +++++++-- assets/compiled/protobuf_version.h | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 308aa942884..b9c9812d374 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,8 +76,7 @@ jobs: with: run: | set -e - make -C assets clean - make -C assets + make assets_manifest - name: 'Build the firmware in docker' uses: ./.github/actions/docker @@ -117,7 +116,6 @@ jobs: - name: 'Bundle resources' if: ${{ !github.event.pull_request.head.repo.fork }} run: | - ./scripts/assets.py manifest assets/resources tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources - name: 'Bundle core2 firmware' diff --git a/Makefile b/Makefile index 59f5a4def9c..1690d028ebb 100644 --- a/Makefile +++ b/Makefile @@ -89,14 +89,19 @@ updater_debug: @$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug .PHONY: updater_package_bin -updater_package: firmware_all updater +updater_package_bin: firmware_all updater @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)" .PHONY: updater_package updater_package: firmware_all updater - @$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources @$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)" +.PHONY: assets_manifest +assets_manifest: + @$(MAKE) -C $(PROJECT_ROOT)/assets clean + @$(MAKE) -C $(PROJECT_ROOT)/assets + @$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources + .PHONY: flash_radio flash_radio: @$(PROJECT_ROOT)/scripts/flash.py core2radio 0x080D7000 $(COPRO_DIR)/stm32wb5x_BLE_Stack_light_fw.bin diff --git a/assets/compiled/protobuf_version.h b/assets/compiled/protobuf_version.h index 53e0b73bf7f..0459d12060e 100644 --- a/assets/compiled/protobuf_version.h +++ b/assets/compiled/protobuf_version.h @@ -1,3 +1,3 @@ #pragma once #define PROTOBUF_MAJOR_VERSION 0 -#define PROTOBUF_MINOR_VERSION 5 +#define PROTOBUF_MINOR_VERSION 6 From d5b30eb5c246cd6e345834dfc5b328b1cf3bf461 Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 22:02:56 +0400 Subject: [PATCH 6/8] Removed raw path formatting --- scripts/update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update.py b/scripts/update.py index 83a8d84f7dd..187c9288bbb 100755 --- a/scripts/update.py +++ b/scripts/update.py @@ -76,7 +76,7 @@ def generate(self): else: file.writeKey("Radio CRC", self.int2ffhex(0)) file.writeKey("Assets", assets_basename) - file.save("%s/update.fuf" % self.args.directory) + file.save(join(self.args.directory, "update.fuf")) return 0 From 165825a98dcbc9f85969e3e15aef2bd60ad228dc Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 22:48:19 +0400 Subject: [PATCH 7/8] Visual progress for resource extraction --- lib/toolbox/tar/tar_archive.c | 42 ++++++++++++++++++++++++++++++----- lib/toolbox/tar/tar_archive.h | 7 ++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c index c85bfeea61d..e3cc1818ecc 100644 --- a/lib/toolbox/tar/tar_archive.c +++ b/lib/toolbox/tar/tar_archive.c @@ -15,6 +15,8 @@ typedef struct TarArchive { Storage* storage; mtar_t tar; + tar_unpack_file_cb unpack_cb; + void* unpack_cb_context; } TarArchive; /* API WRAPPER */ @@ -51,6 +53,7 @@ TarArchive* tar_archive_alloc(Storage* storage) { furi_check(storage); TarArchive* archive = malloc(sizeof(TarArchive)); archive->storage = storage; + archive->unpack_cb = NULL; return archive; } @@ -92,6 +95,28 @@ void tar_archive_free(TarArchive* archive) { } } +void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context) { + furi_assert(archive); + archive->unpack_cb = callback; + archive->unpack_cb_context = context; +} + +static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) { + UNUSED(tar); + UNUSED(header); + int32_t* counter = param; + (*counter)++; + return 0; +} + +int32_t tar_archive_get_entries_count(TarArchive* archive) { + int32_t counter = 0; + if(mtar_foreach(&archive->tar, tar_archive_entry_counter, &counter) != MTAR_ESUCCESS) { + counter = -1; + } + return counter; +} + bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) { furi_assert(archive); return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS); @@ -142,9 +167,17 @@ typedef struct { static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) { TarArchiveDirectoryOpParams* op_params = param; + TarArchive* archive = op_params->archive; string_t fname; - if(strcmp(header->name, ".") == 0) { + bool skip_entry = false; + if(archive->unpack_cb) { + skip_entry = !archive->unpack_cb( + header->name, header->type == MTAR_TDIR, archive->unpack_cb_context); + } + + if(skip_entry) { + FURI_LOG_W(TAG, "filter: skipping entry \"%s\"", header->name); return 0; } @@ -152,8 +185,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, string_init(fname); path_concat(op_params->work_dir, header->name, fname); - bool create_res = - storage_simply_mkdir(op_params->archive->storage, string_get_cstr(fname)); + bool create_res = storage_simply_mkdir(archive->storage, string_get_cstr(fname)); string_clear(fname); return create_res ? 0 : -1; } @@ -166,7 +198,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, string_init(fname); path_concat(op_params->work_dir, header->name, fname); FURI_LOG_I(TAG, "Extracting %d bytes to '%s'", header->size, header->name); - File* out_file = storage_file_alloc(op_params->archive->storage); + File* out_file = storage_file_alloc(archive->storage); uint8_t* readbuf = malloc(FILE_BLOCK_SIZE); bool failed = false; @@ -307,4 +339,4 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch free(name); storage_file_free(directory); return success; -} \ No newline at end of file +} diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h index fe3e248ea8c..a8976181fb3 100644 --- a/lib/toolbox/tar/tar_archive.h +++ b/lib/toolbox/tar/tar_archive.h @@ -34,6 +34,13 @@ bool tar_archive_add_file( bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix); +int32_t tar_archive_get_entries_count(TarArchive* archive); + +/* Optional per-entry callback on unpacking - return false to skip entry */ +typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context); + +void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context); + /* Low-level API */ bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath); From ec4c708181ab612867f4e8d68928275762e400da Mon Sep 17 00:00:00 2001 From: hedger Date: Mon, 18 Apr 2022 22:52:33 +0400 Subject: [PATCH 8/8] Renamed update status enum --- applications/updater/util/update_task.c | 2 +- applications/updater/util/update_task.h | 2 +- .../updater/util/update_task_workers.c | 42 +++++++++++++++---- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/applications/updater/util/update_task.c b/applications/updater/util/update_task.c index 197d42f5d3c..04405bcfd64 100644 --- a/applications/updater/util/update_task.c +++ b/applications/updater/util/update_task.c @@ -19,7 +19,7 @@ static const char* update_task_stage_descr[] = { [UpdateTaskStageRadioCommit] = "Applying radio stack", [UpdateTaskStageLfsBackup] = "Backing up LFS", [UpdateTaskStageLfsRestore] = "Restoring LFS", - [UpdateTaskStageResourcesUpdate] = "Updating resources", + [UpdateTaskStageAssetsUpdate] = "Updating assets", [UpdateTaskStageCompleted] = "Completed!", [UpdateTaskStageError] = "Error", }; diff --git a/applications/updater/util/update_task.h b/applications/updater/util/update_task.h index 2325824cb02..25650de8787 100644 --- a/applications/updater/util/update_task.h +++ b/applications/updater/util/update_task.h @@ -23,7 +23,7 @@ typedef enum { UpdateTaskStageRadioCommit, UpdateTaskStageLfsBackup, UpdateTaskStageLfsRestore, - UpdateTaskStageResourcesUpdate, + UpdateTaskStageAssetsUpdate, UpdateTaskStageCompleted, UpdateTaskStageError, } UpdateTaskStage; diff --git a/applications/updater/util/update_task_workers.c b/applications/updater/util/update_task_workers.c index 31f018dcf43..2a22692bcbb 100644 --- a/applications/updater/util/update_task_workers.c +++ b/applications/updater/util/update_task_workers.c @@ -120,6 +120,23 @@ static bool update_task_pre_update(UpdateTask* update_task) { return success; } +typedef struct { + UpdateTask* update_task; + int32_t total_files, processed_files; +} TarUnpackProgress; + +static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) { + UNUSED(name); + UNUSED(is_directory); + TarUnpackProgress* unpack_progress = context; + unpack_progress->processed_files++; + update_task_set_progress( + unpack_progress->update_task, + UpdateTaskStageProgress, + unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1)); + return true; +} + static bool update_task_post_update(UpdateTask* update_task) { bool success = false; @@ -133,8 +150,8 @@ static bool update_task_post_update(UpdateTask* update_task) { path_concat( string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path); - bool unpack_resouces = !string_empty_p(update_task->manifest->resource_bundle); - if(unpack_resouces) { + bool unpack_resources = !string_empty_p(update_task->manifest->resource_bundle); + if(unpack_resources) { update_task->state.total_stages++; } @@ -143,18 +160,29 @@ static bool update_task_post_update(UpdateTask* update_task) { CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path))); - if(unpack_resouces) { - update_task_set_progress(update_task, UpdateTaskStageResourcesUpdate, 0); + if(unpack_resources) { + TarUnpackProgress progress = { + .update_task = update_task, + .total_files = 0, + .processed_files = 0, + }; + update_task_set_progress(update_task, UpdateTaskStageAssetsUpdate, 0); path_concat( string_get_cstr(update_task->update_path), string_get_cstr(update_task->manifest->resource_bundle), file_path); + update_task_set_progress(update_task, UpdateTaskStageProgress, 0); TarArchive* archive = tar_archive_alloc(update_task->storage); - update_task_set_progress(update_task, UpdateTaskStageProgress, 10); - success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ) && - tar_archive_unpack_to(archive, EXT_PATH); + tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress); + success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ); + if(success) { + progress.total_files = tar_archive_get_entries_count(archive); + if(progress.total_files > 0) { + tar_archive_unpack_to(archive, EXT_PATH); + } + } tar_archive_free(archive); } } while(false);