fbt: reproducible manifest builds & improvements (flipperdevices#1801)

* fbt: reproducible manifest builds, less rebuild on small updates; scripts: assets: using timestamp from commandline af available
* fbt: added app import validation for launch_app & single app build targets
* fbt: COMSTR for app imports validation
* docs: minor fixes
* docs: markdown fix
* vscode: comments for RTOS startup

Co-authored-by: あく <[email protected]>

.vscode/example/launch.json

@@ -28,13 +28,14 @@
             "servertype": "openocd",
             "device": "stlink",
             "svdFile": "./debug/STM32WB55_CM4.svd",
+            // If you're debugging early in the boot process, before OS scheduler is running,
+            // you have to comment out the following line.
             "rtos": "FreeRTOS",
             "configFiles": [
                 "interface/stlink.cfg",
                 "./debug/stm32wbx.cfg",
             ],
             "postAttachCommands": [
-                // "attach 1",
                 // "compare-sections",
                 "source debug/flipperapps.py",
                 // "source debug/FreeRTOS/FreeRTOS.py",

assets/SConscript

@@ -1,8 +1,11 @@
 Import("env")
 
+from fbt.version import get_git_commit_unix_timestamp
+
 assetsenv = env.Clone(
     tools=["fbt_assets"],
     FW_LIB_NAME="assets",
+    GIT_UNIX_TIMESTAMP=get_git_commit_unix_timestamp(),
 )
 assetsenv.ApplyLibFlags()
 
@@ -90,10 +93,11 @@ if assetsenv["IS_BASE_FIRMWARE"]:
         "#/assets/resources/Manifest",
         assetsenv.GlobRecursive("*", "resources", exclude="Manifest"),
         action=Action(
-            '${PYTHON3} "${ASSETS_COMPILER}" manifest "${TARGET.dir.posix}"',
+            '${PYTHON3} "${ASSETS_COMPILER}" manifest "${TARGET.dir.posix}" --timestamp=${GIT_UNIX_TIMESTAMP}',
             "${RESMANIFESTCOMSTR}",
         ),
     )
+    assetsenv.Precious(resources)
     assetsenv.AlwaysBuild(resources)
     assetsenv.Clean(
         resources,

documentation/AppManifests.md

@@ -43,8 +43,8 @@ Only 2 parameters are mandatory: ***appid*** and ***apptype***, others are optio
 
 The following parameters are used only for [FAPs](./AppsOnSDCard.md):
 
-* **sources**: list of file name masks, used for gathering sources within app folder. Default value of ["\*.c\*"] includes C and CPP source files.
-* **fap_version**: string, 2 numbers in form of "x.y": application version to be embedded within .fap file.
+* **sources**: list of strings, file name masks, used for gathering sources within app folder. Default value of `["*.c*"]` includes C and CPP source files.
+* **fap_version**: tuple, 2 numbers in form of (x,y): application version to be embedded within .fap file. Default value is (0,1), meanig version "0.1".
 * **fap_icon**: name of a .png file, 1-bit color depth, 10x10px, to be embedded within .fap file.
 * **fap_libs**: list of extra libraries to link application against. Provides access to extra functions that are not exported as a part of main firmware at expense of increased .fap file size and RAM consumption.
 * **fap_category**: string, may be empty. App subcategory, also works as path of FAP within apps folder in the file system.

firmware.scons

@@ -96,6 +96,7 @@ if not env["VERBOSE"]:
         SDKSYM_GENERATOR_COMSTR="\tSDKSYM\t${TARGET}",
         APPMETA_COMSTR="\tAPPMETA\t${TARGET}",
         APPMETAEMBED_COMSTR="\tFAP\t${TARGET}",
+        APPCHECK_COMSTR="\tAPPCHK\t${SOURCE}",
     )
 
 

scripts/assets.py

@@ -39,6 +39,13 @@ def init(self):
             "manifest", help="Create directory Manifest"
         )
         self.parser_manifest.add_argument("local_path", help="local_path")
+        self.parser_manifest.add_argument(
+            "--timestamp",
+            help="timestamp value to embed",
+            default=0,
+            type=int,
+            required=False,
+        )
         self.parser_manifest.set_defaults(func=self.manifest)
 
         self.parser_copro = self.subparsers.add_parser(
@@ -213,7 +220,7 @@ def manifest(self):
         self.logger.info(
             f'Creating temporary Manifest for directory "{directory_path}"'
         )
-        new_manifest = Manifest()
+        new_manifest = Manifest(self.args.timestamp)
         new_manifest.create(directory_path)
 
         self.logger.info(f"Comparing new manifest with existing")

scripts/flipper/assets/manifest.py

@@ -106,11 +106,11 @@ def toLine(self):
 
 
 class Manifest:
-    def __init__(self):
+    def __init__(self, timestamp_value=None):
         self.version = None
         self.records = []
         self.records.append(ManifestRecordVersion(MANIFEST_VERSION))
-        self.records.append(ManifestRecordTimestamp(timestamp()))
+        self.records.append(ManifestRecordTimestamp(timestamp_value or timestamp()))
         self.logger = logging.getLogger(self.__class__.__name__)
 
     def load(self, filename):

site_scons/extapps.scons

@@ -86,12 +86,13 @@ if appenv["FORCE"]:
 Alias(appenv["FIRMWARE_BUILD_CFG"] + "_extapps", extapps["compact"].values())
 
 if appsrc := appenv.subst("$APPSRC"):
-    app_manifest, fap_file = appenv.GetExtAppFromPath(appsrc)
+    app_manifest, fap_file, app_validator = appenv.GetExtAppFromPath(appsrc)
     appenv.PhonyTarget(
         "launch_app",
         '${PYTHON3} scripts/runfap.py ${SOURCE} --fap_dst_dir "/ext/apps/${FAP_CATEGORY}"',
         source=fap_file,
         FAP_CATEGORY=app_manifest.fap_category,
     )
+    appenv.Alias("launch_app", app_validator)
 
 Return("extapps")

site_scons/fbt/appmanifest.py

@@ -38,7 +38,7 @@ class FlipperApplication:
     sdk_headers: List[str] = field(default_factory=list)
     # .fap-specific
     sources: List[str] = field(default_factory=lambda: ["*.c*"])
-    fap_version: Tuple[int] = field(default_factory=lambda: (0, 0))
+    fap_version: Tuple[int] = field(default_factory=lambda: (0, 1))
     fap_icon: Optional[str] = None
     fap_libs: List[str] = field(default_factory=list)
     fap_category: str = ""

site_scons/fbt/version.py

@@ -3,6 +3,11 @@
 from functools import cache
 
 
+@cache
+def get_git_commit_unix_timestamp():
+    return int(subprocess.check_output(["git", "show", "-s", "--format=%ct"]))
+
+
 @cache
 def get_fast_git_version_id():
     try:

site_scons/site_tools/fbt_extapps.py

@@ -37,7 +37,15 @@ def BuildAppElf(env, app):
         APP=app,
     )
 
-    env.Depends(app_elf_augmented, [env["SDK_DEFINITION"], env.Value(app)])
+    manifest_vals = vars(app)
+    manifest_vals = {
+        k: v for k, v in manifest_vals.items() if k not in ("_appdir", "_apppath")
+    }
+
+    env.Depends(
+        app_elf_augmented,
+        [env["SDK_DEFINITION"], env.Value(manifest_vals)],
+    )
     if app.fap_icon:
         env.Depends(
             app_elf_augmented,
@@ -47,6 +55,7 @@ def BuildAppElf(env, app):
 
     app_elf_import_validator = env.ValidateAppImports(app_elf_augmented)
     env.AlwaysBuild(app_elf_import_validator)
+    env.Alias(app_alias, app_elf_import_validator)
     return (app_elf_augmented, app_elf_raw, app_elf_import_validator)
 
 
@@ -100,9 +109,13 @@ def GetExtAppFromPath(env, app_dir):
 
     app_elf = env["_extapps"]["compact"].get(app.appid, None)
     if not app_elf:
-        raise UserError(f"No external app found for {app.appid}")
+        raise UserError(
+            f"Application {app.appid} is not configured for building as external"
+        )
+
+    app_validator = env["_extapps"]["validators"].get(app.appid, None)
 
-    return (app, app_elf[0])
+    return (app, app_elf[0], app_validator[0])
 
 
 def generate(env, **kw):
@@ -138,7 +151,7 @@ def generate(env, **kw):
                     ),
                     Action(
                         validate_app_imports,
-                        None,  # "$APPCHECK_COMSTR",
+                        "$APPCHECK_COMSTR",
                     ),
                 ],
                 suffix=".impsyms",