[FL-2477] Updater support for resource bundles (#1131)

* Resource unpacking core * Added more fields to manifest; updated dist scripts * Python linter fixes * Parsing manifest before unpacking * Updated pipelines for separate resource build * Removed raw path formatting * Visual progress for resource extraction * Renamed update status enum Co-authored-by: Aleksandr Kutuzov <[email protected]>
flipperdevices · Apr 19, 2022 · e8499e4 · e8499e4
1 parent 1623134
commit e8499e4
Show file tree

Hide file tree

Showing 12 changed files with 260 additions and 47 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -76,8 +76,7 @@ jobs:
         with:
           run: |
             set -e
-            make -C assets clean
-            make -C assets
+            make assets_manifest
             git diff --quiet || ( echo "Assets recompilation required."; exit 255 )
 
       - name: 'Build the firmware in docker'
@@ -118,7 +117,6 @@ jobs:
       - name: 'Bundle resources'
         if: ${{ !github.event.pull_request.head.repo.fork }}
         run: |
-          ./scripts/assets.py manifest assets/resources
           tar czpf artifacts/flipper-z-any-resources-${{steps.names.outputs.suffix}}.tgz -C assets resources
 
       - name: 'Bundle core2 firmware'

diff --git a/Makefile b/Makefile
@@ -92,9 +92,19 @@ updater_clean:
 updater_debug:
 	@$(MAKE) -C $(PROJECT_ROOT)/firmware -j$(NPROCS) RAM_EXEC=1 debug
 
+.PHONY: updater_package_bin
+updater_package_bin: firmware_all updater
+	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
+
 .PHONY: updater_package
 updater_package: firmware_all updater
-	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) --bundlever "$(VERSION_STRING)"
+	@$(PROJECT_ROOT)/scripts/dist.py copy -t $(TARGET) -p firmware updater -s $(DIST_SUFFIX) -a assets/resources --bundlever "$(VERSION_STRING)"
+
+.PHONY: assets_manifest
+assets_manifest:
+	@$(MAKE) -C $(PROJECT_ROOT)/assets clean
+	@$(MAKE) -C $(PROJECT_ROOT)/assets
+	@$(PROJECT_ROOT)/scripts/assets.py manifest assets/resources
 
 .PHONY: flash_radio
 flash_radio:

diff --git a/applications/updater/scenes/updater_scene_main.c b/applications/updater/scenes/updater_scene_main.c
@@ -73,7 +73,7 @@ bool updater_scene_main_on_event(void* context, SceneManagerEvent event) {
 
         case UpdaterCustomEventRetryUpdate:
             if(!update_task_is_running(updater->update_task) &&
-               (update_task_get_state(updater->update_task)->stage != UpdateTaskStageComplete))
+               (update_task_get_state(updater->update_task)->stage != UpdateTaskStageCompleted))
                 update_task_start(updater->update_task);
             consumed = true;
             break;

diff --git a/applications/updater/util/update_task.c b/applications/updater/util/update_task.c
@@ -19,7 +19,8 @@ static const char* update_task_stage_descr[] = {
     [UpdateTaskStageRadioCommit] = "Applying radio stack",
     [UpdateTaskStageLfsBackup] = "Backing up LFS",
     [UpdateTaskStageLfsRestore] = "Restoring LFS",
-    [UpdateTaskStageComplete] = "Complete",
+    [UpdateTaskStageAssetsUpdate] = "Updating assets",
+    [UpdateTaskStageCompleted] = "Completed!",
     [UpdateTaskStageError] = "Error",
 };
 

diff --git a/applications/updater/util/update_task.h b/applications/updater/util/update_task.h
@@ -23,7 +23,8 @@ typedef enum {
     UpdateTaskStageRadioCommit,
     UpdateTaskStageLfsBackup,
     UpdateTaskStageLfsRestore,
-    UpdateTaskStageComplete,
+    UpdateTaskStageAssetsUpdate,
+    UpdateTaskStageCompleted,
     UpdateTaskStageError,
 } UpdateTaskStage;
 

diff --git a/applications/updater/util/update_task_workers.c b/applications/updater/util/update_task_workers.c
@@ -8,6 +8,7 @@
 #include <update_util/dfu_file.h>
 #include <update_util/lfs_backup.h>
 #include <update_util/update_operation.h>
+#include <toolbox/tar/tar_archive.h>
 
 #define CHECK_RESULT(x) \
     if(!(x)) {          \
@@ -19,6 +20,8 @@
 /* Written into DFU file by build pipeline */
 #define FLIPPER_ZERO_DFU_DEVICE_CODE 0xFFFF
 
+#define EXT_PATH "/ext"
+
 static const DfuValidationParams flipper_dfu_params = {
     .device = FLIPPER_ZERO_DFU_DEVICE_CODE,
     .product = STM_DFU_PRODUCT_ID,
@@ -85,7 +88,7 @@ int32_t update_task_worker_flash_writer(void* context) {
             CHECK_RESULT(dfu_file_process_targets(&page_task, update_task->file, valid_targets));
         }
 
-        update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
+        update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
 
         furi_hal_rtc_set_boot_mode(FuriHalRtcBootModePostUpdate);
 
@@ -99,6 +102,95 @@ int32_t update_task_worker_flash_writer(void* context) {
     return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
 }
 
+static bool update_task_pre_update(UpdateTask* update_task) {
+    bool success = false;
+    string_t backup_file_path;
+    string_init(backup_file_path);
+    path_concat(
+        string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
+
+    update_task->state.total_stages = 1;
+    update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
+    furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
+    if((success = lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
+        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
+    }
+
+    string_clear(backup_file_path);
+    return success;
+}
+
+typedef struct {
+    UpdateTask* update_task;
+    int32_t total_files, processed_files;
+} TarUnpackProgress;
+
+static bool update_task_resource_unpack_cb(const char* name, bool is_directory, void* context) {
+    UNUSED(name);
+    UNUSED(is_directory);
+    TarUnpackProgress* unpack_progress = context;
+    unpack_progress->processed_files++;
+    update_task_set_progress(
+        unpack_progress->update_task,
+        UpdateTaskStageProgress,
+        unpack_progress->processed_files * 100 / (unpack_progress->total_files + 1));
+    return true;
+}
+
+static bool update_task_post_update(UpdateTask* update_task) {
+    bool success = false;
+
+    string_t file_path;
+    string_init(file_path);
+
+    update_task->state.total_stages = 2;
+
+    do {
+        CHECK_RESULT(update_task_parse_manifest(update_task));
+        path_concat(
+            string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, file_path);
+
+        bool unpack_resources = !string_empty_p(update_task->manifest->resource_bundle);
+        if(unpack_resources) {
+            update_task->state.total_stages++;
+        }
+
+        update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
+        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
+
+        CHECK_RESULT(lfs_backup_unpack(update_task->storage, string_get_cstr(file_path)));
+
+        if(unpack_resources) {
+            TarUnpackProgress progress = {
+                .update_task = update_task,
+                .total_files = 0,
+                .processed_files = 0,
+            };
+            update_task_set_progress(update_task, UpdateTaskStageAssetsUpdate, 0);
+
+            path_concat(
+                string_get_cstr(update_task->update_path),
+                string_get_cstr(update_task->manifest->resource_bundle),
+                file_path);
+
+            update_task_set_progress(update_task, UpdateTaskStageProgress, 0);
+            TarArchive* archive = tar_archive_alloc(update_task->storage);
+            tar_archive_set_file_callback(archive, update_task_resource_unpack_cb, &progress);
+            success = tar_archive_open(archive, string_get_cstr(file_path), TAR_OPEN_MODE_READ);
+            if(success) {
+                progress.total_files = tar_archive_get_entries_count(archive);
+                if(progress.total_files > 0) {
+                    tar_archive_unpack_to(archive, EXT_PATH);
+                }
+            }
+            tar_archive_free(archive);
+        }
+    } while(false);
+
+    string_clear(file_path);
+    return success;
+}
+
 int32_t update_task_worker_backup_restore(void* context) {
     furi_assert(context);
     UpdateTask* update_task = context;
@@ -112,37 +204,22 @@ int32_t update_task_worker_backup_restore(void* context) {
     }
 
     update_task->state.current_stage_idx = 0;
-    update_task->state.total_stages = 1;
 
     if(!update_operation_get_current_package_path(update_task->storage, update_task->update_path)) {
         return UPDATE_TASK_FAILED;
     }
 
-    string_t backup_file_path;
-    string_init(backup_file_path);
-    path_concat(
-        string_get_cstr(update_task->update_path), LFS_BACKUP_DEFAULT_FILENAME, backup_file_path);
-
     if(boot_mode == FuriHalRtcBootModePreUpdate) {
-        update_task_set_progress(update_task, UpdateTaskStageLfsBackup, 0);
-        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal); // to avoid bootloops
-        if((success =
-                lfs_backup_create(update_task->storage, string_get_cstr(backup_file_path)))) {
-            furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeUpdate);
-        }
+        success = update_task_pre_update(update_task);
     } else if(boot_mode == FuriHalRtcBootModePostUpdate) {
-        update_task_set_progress(update_task, UpdateTaskStageLfsRestore, 0);
-        furi_hal_rtc_set_boot_mode(FuriHalRtcBootModeNormal);
-        success = lfs_backup_unpack(update_task->storage, string_get_cstr(backup_file_path));
+        success = update_task_post_update(update_task);
     }
 
     if(success) {
-        update_task_set_progress(update_task, UpdateTaskStageComplete, 100);
+        update_task_set_progress(update_task, UpdateTaskStageCompleted, 100);
     } else {
         update_task_set_progress(update_task, UpdateTaskStageError, update_task->state.progress);
     }
 
-    string_clear(backup_file_path);
-
     return success ? UPDATE_TASK_NOERR : UPDATE_TASK_FAILED;
-}
+}
diff --git a/lib/toolbox/tar/tar_archive.c b/lib/toolbox/tar/tar_archive.c
@@ -15,6 +15,8 @@
 typedef struct TarArchive {
     Storage* storage;
     mtar_t tar;
+    tar_unpack_file_cb unpack_cb;
+    void* unpack_cb_context;
 } TarArchive;
 
 /* API WRAPPER */
@@ -51,6 +53,7 @@ TarArchive* tar_archive_alloc(Storage* storage) {
     furi_check(storage);
     TarArchive* archive = malloc(sizeof(TarArchive));
     archive->storage = storage;
+    archive->unpack_cb = NULL;
     return archive;
 }
 
@@ -92,6 +95,28 @@ void tar_archive_free(TarArchive* archive) {
     }
 }
 
+void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context) {
+    furi_assert(archive);
+    archive->unpack_cb = callback;
+    archive->unpack_cb_context = context;
+}
+
+static int tar_archive_entry_counter(mtar_t* tar, const mtar_header_t* header, void* param) {
+    UNUSED(tar);
+    UNUSED(header);
+    int32_t* counter = param;
+    (*counter)++;
+    return 0;
+}
+
+int32_t tar_archive_get_entries_count(TarArchive* archive) {
+    int32_t counter = 0;
+    if(mtar_foreach(&archive->tar, tar_archive_entry_counter, &counter) != MTAR_ESUCCESS) {
+        counter = -1;
+    }
+    return counter;
+}
+
 bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath) {
     furi_assert(archive);
     return (mtar_write_dir_header(&archive->tar, dirpath) == MTAR_ESUCCESS);
@@ -142,14 +167,25 @@ typedef struct {
 
 static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header, void* param) {
     TarArchiveDirectoryOpParams* op_params = param;
+    TarArchive* archive = op_params->archive;
     string_t fname;
 
+    bool skip_entry = false;
+    if(archive->unpack_cb) {
+        skip_entry = !archive->unpack_cb(
+            header->name, header->type == MTAR_TDIR, archive->unpack_cb_context);
+    }
+
+    if(skip_entry) {
+        FURI_LOG_W(TAG, "filter: skipping entry \"%s\"", header->name);
+        return 0;
+    }
+
     if(header->type == MTAR_TDIR) {
         string_init(fname);
         path_concat(op_params->work_dir, header->name, fname);
 
-        bool create_res =
-            storage_simply_mkdir(op_params->archive->storage, string_get_cstr(fname));
+        bool create_res = storage_simply_mkdir(archive->storage, string_get_cstr(fname));
         string_clear(fname);
         return create_res ? 0 : -1;
     }
@@ -162,7 +198,7 @@ static int archive_extract_foreach_cb(mtar_t* tar, const mtar_header_t* header,
     string_init(fname);
     path_concat(op_params->work_dir, header->name, fname);
     FURI_LOG_I(TAG, "Extracting %d bytes to '%s'", header->size, header->name);
-    File* out_file = storage_file_alloc(op_params->archive->storage);
+    File* out_file = storage_file_alloc(archive->storage);
     uint8_t* readbuf = malloc(FILE_BLOCK_SIZE);
 
     bool failed = false;
@@ -303,4 +339,4 @@ bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const ch
     free(name);
     storage_file_free(directory);
     return success;
-}
+}
diff --git a/lib/toolbox/tar/tar_archive.h b/lib/toolbox/tar/tar_archive.h
@@ -34,6 +34,13 @@ bool tar_archive_add_file(
 
 bool tar_archive_add_dir(TarArchive* archive, const char* fs_full_path, const char* path_prefix);
 
+int32_t tar_archive_get_entries_count(TarArchive* archive);
+
+/* Optional per-entry callback on unpacking - return false to skip entry */
+typedef bool (*tar_unpack_file_cb)(const char* name, bool is_directory, void* context);
+
+void tar_archive_set_file_callback(TarArchive* archive, tar_unpack_file_cb callback, void* context);
+
 /* Low-level API */
 bool tar_archive_dir_add_element(TarArchive* archive, const char* dirpath);