fbt fixes & improvements (#1490)

* fbt: minimal USB flash mode; scripts: faster storage.py with larger chunks
* fbt: fixed creation of temporary file nodes confusing scons
* docs: removed refs to --with-updater
* fbt: removed splashscreen from minimal update package
* fbt: renamed dist arguments for consistency
* docs: fixed updater_debug target
* fbt: separate target for generating compilation_database.json without building the code.
* fbt: added `jflash` target for programming over JLink probe; refactored usb flashing targets
* fbt: building updater_app in unit_tests configuration
* fbt: fixed reset behavior after flashing with J-Link
* fbt: generating .map file for firmware binary & external apps
* fbt/core: moved library contents before apps code

Co-authored-by: あく <[email protected]>

.github/workflows/build.yml

@@ -78,7 +78,7 @@ jobs:
             set -e
             for TARGET in ${TARGETS}
             do
-              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` --with-updater updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
+              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package ${{ startsWith(github.ref, 'refs/tags') && 'DEBUG=0 COMPACT=1' || '' }}
             done
 
       - name: 'Move upload files'
@@ -214,5 +214,5 @@ jobs:
             set -e
             for TARGET in ${TARGETS}
             do
-              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` --with-updater updater_package DEBUG=0 COMPACT=1
+              ./fbt TARGET_HW=`echo ${TARGET} | sed 's/f//'` updater_package DEBUG=0 COMPACT=1
             done

ReadMe.md

@@ -29,11 +29,11 @@ They both must be flashed in the order described.
 
 With Flipper attached over USB:
 
-`./fbt --with-updater flash_usb`
+`./fbt flash_usb`
 
 Just building the package:
 
-`./fbt --with-updater updater_package`
+`./fbt updater_package`
 
 To update, copy the resulting directory to Flipper's SD card and navigate to `update.fuf` file in Archive app. 
 

SConstruct

@@ -33,8 +33,10 @@ coreenv["ROOT_DIR"] = Dir(".")
 
 # Create a separate "dist" environment and add construction envs to it
 distenv = coreenv.Clone(
-    tools=["fbt_dist", "openocd", "blackmagic"],
-    OPENOCD_GDB_PIPE=["|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"],
+    tools=["fbt_dist", "openocd", "blackmagic", "jflash"],
+    OPENOCD_GDB_PIPE=[
+        "|openocd -c 'gdb_port pipe; log_output debug/openocd.log' ${[SINGLEQUOTEFUNC(OPENOCD_OPTS)]}"
+    ],
     GDBOPTS_BASE=[
         "-ex",
         "target extended-remote ${GDBREMOTE}",
@@ -61,6 +63,7 @@ distenv = coreenv.Clone(
         "-ex",
         "compare-sections",
     ],
+    JFLASHPROJECT="${ROOT_DIR.abspath}/debug/fw.jflash",
     ENV=os.environ,
 )
 
@@ -71,19 +74,21 @@ firmware_env = distenv.AddFwProject(
 )
 
 # If enabled, initialize updater-related targets
-if GetOption("fullenv"):
+if GetOption("fullenv") or any(
+    filter(lambda target: "updater" in target or "flash_usb" in target, BUILD_TARGETS)
+):
     updater_env = distenv.AddFwProject(
         base_env=coreenv,
         fw_type="updater",
         fw_env_key="UPD_ENV",
     )
 
     # Target for self-update package
-    dist_arguments = [
-        "-r",
-        '"${ROOT_DIR.abspath}/assets/resources"',
+    dist_basic_arguments = [
         "--bundlever",
         '"${UPDATE_VERSION_STRING}"',
+    ]
+    dist_radio_arguments = [
         "--radio",
         '"${ROOT_DIR.abspath}/${COPRO_STACK_BIN_DIR}/${COPRO_STACK_BIN}"',
         "--radiotype",
@@ -92,16 +97,34 @@ if GetOption("fullenv"):
         "--obdata",
         '"${ROOT_DIR.abspath}/${COPRO_OB_DATA}"',
     ]
-    if distenv["UPDATE_SPLASH"]:
-        dist_arguments += [
+    dist_resource_arguments = [
+        "-r",
+        '"${ROOT_DIR.abspath}/assets/resources"',
+    ]
+    dist_splash_arguments = (
+        [
             "--splash",
             distenv.subst("assets/slideshow/$UPDATE_SPLASH"),
         ]
+        if distenv["UPDATE_SPLASH"]
+        else []
+    )
 
     selfupdate_dist = distenv.DistCommand(
         "updater_package",
         (distenv["DIST_DEPENDS"], firmware_env["FW_RESOURCES"]),
-        DIST_EXTRA=dist_arguments,
+        DIST_EXTRA=[
+            *dist_basic_arguments,
+            *dist_radio_arguments,
+            *dist_resource_arguments,
+            *dist_splash_arguments,
+        ],
+    )
+
+    selfupdate_min_dist = distenv.DistCommand(
+        "updater_minpackage",
+        distenv["DIST_DEPENDS"],
+        DIST_EXTRA=dist_basic_arguments,
     )
 
     # Updater debug
@@ -121,18 +144,16 @@ if GetOption("fullenv"):
     )
 
     # Installation over USB & CLI
-    usb_update_package = distenv.UsbInstall(
-        "#build/usbinstall.flag",
-        (
-            distenv["DIST_DEPENDS"],
-            firmware_env["FW_RESOURCES"],
-            selfupdate_dist,
-        ),
+    usb_update_package = distenv.AddUsbFlashTarget(
+        "#build/usbinstall.flag", (firmware_env["FW_RESOURCES"], selfupdate_dist)
     )
-    if distenv["FORCE"]:
-        distenv.AlwaysBuild(usb_update_package)
-    distenv.Depends(usb_update_package, selfupdate_dist)
-    distenv.Alias("flash_usb", usb_update_package)
+    distenv.Alias("flash_usb_full", usb_update_package)
+
+    usb_minupdate_package = distenv.AddUsbFlashTarget(
+        "#build/minusbinstall.flag", (selfupdate_min_dist,)
+    )
+    distenv.Alias("flash_usb", usb_minupdate_package)
+
 
 # Target for copying & renaming binaries to dist folder
 basic_dist = distenv.DistCommand("fw_dist", distenv["DIST_DEPENDS"])
@@ -147,8 +168,9 @@ distenv.Alias("copro_dist", copro_dist)
 
 firmware_flash = distenv.AddOpenOCDFlashTarget(firmware_env)
 distenv.Alias("flash", firmware_flash)
-if distenv["FORCE"]:
-    distenv.AlwaysBuild(firmware_flash)
+
+firmware_jflash = distenv.AddJFlashTarget(firmware_env)
+distenv.Alias("jflash", firmware_jflash)
 
 firmware_bm_flash = distenv.PhonyTarget(
     "flash_blackmagic",

applications/extapps.scons

@@ -38,6 +38,8 @@ appenv.AppendUnique(
         "-Wl,--no-export-dynamic",
         "-fvisibility=hidden",
         "-Wl,-e${APP_ENTRY}",
+        "-Xlinker",
+        "-Map=${TARGET}.map",
     ],
 )
 

debug/fw.jflash

@@ -0,0 +1,90 @@
+  AppVersion = 76803
+  FileVersion = 2
+[GENERAL]
+  aATEModuleSel[24] = 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+  ConnectMode = 0
+  CurrentFile = "..\build\latest\firmware.bin"
+  DataFileSAddr = 0x08000000
+  GUIMode = 0
+  HostName = ""
+  TargetIF = 1
+  USBPort = 0
+  USBSerialNo = 0x00000000
+  UseATEModuleSelection = 0
+[JTAG]
+  IRLen = 0
+  MultipleTargets = 0
+  NumDevices = 0
+  Speed0 = 8000
+  Speed1 = 8000
+  TAP_Number = 0
+  UseAdaptive0 = 0
+  UseAdaptive1 = 0
+  UseMaxSpeed0 = 0
+  UseMaxSpeed1 = 0
+[CPU]
+  NumInitSteps = 2
+  InitStep0_Action = "Reset"
+  InitStep0_Value0 = 0x00000000
+  InitStep0_Value1 = 0x00000000
+  InitStep0_Comment = ""
+  InitStep1_Action = "Halt"
+  InitStep1_Value0 = 0xFFFFFFFF
+  InitStep1_Value1 = 0xFFFFFFFF
+  InitStep1_Comment = ""
+  NumExitSteps = 1
+  ExitStep0_Action = "Reset"
+  ExitStep0_Value0 = 0x00000005
+  ExitStep0_Value1 = 0x00000032
+  ExitStep0_Comment = ""
+  UseScriptFile = 0
+  ScriptFile = ""
+  UseRAM = 1
+  RAMAddr = 0x20000000
+  RAMSize = 0x00030000
+  CheckCoreID = 1
+  CoreID = 0x6BA02477
+  CoreIDMask = 0x0F000FFF
+  UseAutoSpeed = 0x00000001
+  ClockSpeed = 0x00000000
+  EndianMode = 0
+  ChipName = "ST STM32WB55RG"
+[FLASH]
+  aRangeSel[1] = 0-255
+  BankName = "Internal flash"
+  BankSelMode = 1
+  BaseAddr = 0x08000000
+  NumBanks = 1
+[PRODUCTION]
+  AutoPerformsDisconnect = 0
+  AutoPerformsErase = 1
+  AutoPerformsProgram = 1
+  AutoPerformsSecure = 0
+  AutoPerformsStartApp = 1
+  AutoPerformsUnsecure = 0
+  AutoPerformsVerify = 0
+  EnableFixedVTref = 0
+  EnableTargetPower = 0
+  EraseType = 1
+  FixedVTref = 0x00000CE4
+  MonitorVTref = 0
+  MonitorVTrefMax = 0x0000157C
+  MonitorVTrefMin = 0x000003E8
+  OverrideTimeouts = 0
+  ProgramSN = 0
+  SerialFile = ""
+  SNAddr = 0x00000000
+  SNInc = 0x00000001
+  SNLen = 0x00000004
+  SNListFile = ""
+  SNValue = 0x00000001
+  StartAppType = 1
+  TargetPowerDelay = 0x00000014
+  TimeoutErase = 0x00003A98
+  TimeoutProgram = 0x00002710
+  TimeoutVerify = 0x00002710
+  VerifyType = 1
+[PERFORMANCE]
+  DisableSkipBlankDataOnProgram = 0x00000000
+  PerfromBlankCheckPriorEraseChip = 0x00000001
+  PerfromBlankCheckPriorEraseSelectedSectors = 0x00000001

documentation/OTA.md

@@ -110,15 +110,20 @@ Even if something goes wrong, Updater gives you an option to retry failed operat
 
 ## Full package
 
-To build a basic update package, run `./fbt --with-updater COMPACT=1 DEBUG=0 updater_package`
+To build full update package, including firmware, radio stack and resources for SD card, run `./fbt COMPACT=1 DEBUG=0 updater_package`
+
+
+## Minimal package
+
+To build minimal update package, including only firmware, run `./fbt COMPACT=1 DEBUG=0 updater_minpackage`
 
 
 ## Customizing update bundles
 
 Default update packages are built with Bluetooth Light stack. 
 You can pick a different stack, if your firmware version supports it, and build a bundle with it passing stack type and binary name to `fbt`: 
 
-`./fbt --with-updater updater_package COMPACT=1 DEBUG=0 COPRO_OB_DATA=scripts/ob_custradio.data COPRO_STACK_BIN=stm32wb5x_BLE_Stack_full_fw.bin COPRO_STACK_TYPE=ble_full`  
+`./fbt updater_package COMPACT=1 DEBUG=0 COPRO_OB_DATA=scripts/ob_custradio.data COPRO_STACK_BIN=stm32wb5x_BLE_Stack_full_fw.bin COPRO_STACK_TYPE=ble_full`  
 
 Note that `COPRO_OB_DATA` must point to a valid file in `scripts` folder containing reference Option Byte data matching to your radio stack type.
 

documentation/fbt.md

@@ -20,7 +20,7 @@ Make sure that `gcc-arm-none-eabi` toolchain & OpenOCD executables are in system
 
 To build with FBT, call it specifying configuration options & targets to build. For example,
 
-`./fbt --with-updater COMPACT=1 DEBUG=0 VERBOSE=1 updater_package copro_dist`
+`./fbt COMPACT=1 DEBUG=0 VERBOSE=1 updater_package copro_dist`
 
 To run cleanup (think of `make clean`) for specified targets, add `-c` option.
 
@@ -31,13 +31,13 @@ FBT keeps track of internal dependencies, so you only need to build the highest-
 ### High-level (what you most likely need)
 
 - `fw_dist` - build & publish firmware to `dist` folder. This is a default target, when no other are specified
-- `updater_package` - build self-update package. _Requires `--with-updater` option_
+- `updater_package`, `updater_minpackage` - build self-update package. Minimal version only inclues firmware's DFU file; full version also includes radio stack & resources for SD card
 - `copro_dist` - bundle Core2 FUS+stack binaries for qFlipper
 - `flash` - flash attached device with OpenOCD over ST-Link
-- `flash_usb` - build, upload and install update package to device over USB.  _Requires `--with-updater` option_
+- `flash_usb`, `flash_usb_full` - build, upload and install update package to device over USB. See details on `updater_package`, `updater_minpackage` 
 - `debug` - build and flash firmware, then attach with gdb with firmware's .elf loaded
-- `debug_updater` - attach gdb with updater's .elf loaded. _Requires `--with-updater` option_
-- `debug_other` - attach gdb without loading any .elf. Allows to manually add external elf files with `add-symbol-file` in gdb.
+- `debug_other` - attach gdb without loading any .elf. Allows to manually add external elf files with `add-symbol-file` in gdb
+- `updater_debug` - attach gdb with updater's .elf loaded
 - `blackmagic` - debug firmware with Blackmagic probe (WiFi dev board)
 - `openocd` - just start OpenOCD
 - `get_blackmagic` - output blackmagic address in gdb remote format. Useful for IDE integration
@@ -49,9 +49,11 @@ FBT keeps track of internal dependencies, so you only need to build the highest-
     - Check out `--extra-ext-apps` for force adding extra apps to external build 
     - `firmware_snake_game_list`, etc - generate source + assembler listing for app's .elf
 - `flash`, `firmware_flash` - flash current version to attached device with OpenOCD over ST-Link
+- `jflash` - flash current version to attached device with JFlash using J-Link probe. JFlash executable must be on your $PATH
 - `flash_blackmagic` - flash current version to attached device with Blackmagic probe
 - `firmware_all`, `updater_all` - build basic set of binaries
 - `firmware_list`, `updater_list` - generate source + assembler listing
+- `firmware_cdb`, `updater_cdb` - generate `compilation_database.json` file for external tools and IDEs. It can be created without actually building the firmware. 
 
 ### Assets
 
@@ -66,7 +68,7 @@ FBT keeps track of internal dependencies, so you only need to build the highest-
 ## Command-line parameters
 
 - `--options optionfile.py` (default value `fbt_options.py`) - load file with multiple configuration values
-- `--with-updater` - enables updater-related targets and dependency tracking. Enabling this option introduces extra startup time costs, so use it when bundling update packages. Or if you have a fast computer and don't care about a few extra seconds of startup time
+- `--with-updater` - enables updater-related targets and dependency tracking. Enabling this option introduces extra startup time costs, so use it when bundling update packages. _Explicily enabling this should no longer be required, fbt now has specific handling for updater-related targets_
 - `--extra-int-apps=app1,app2,appN` - forces listed apps to be built as internal with `firmware` target
 - `--extra-ext-apps=app1,app2,appN` - forces listed apps to be built as external with `firmware_extapps` target
 

fbt_options.py

@@ -80,6 +80,7 @@
     ],
     "unit_tests": [
         "basic_services",
+        "updater_app",
         "unit_tests",
     ],
 }

firmware.scons

@@ -9,7 +9,11 @@ from fbt.util import (
 
 # Building initial C environment for libs
 env = ENV.Clone(
-    tools=["compilation_db", "fwbin", "fbt_apps"],
+    tools=[
+        ("compilation_db", {"COMPILATIONDB_COMSTR": "\tCDB\t${TARGET}"}),
+        "fwbin",
+        "fbt_apps",
+    ],
     COMPILATIONDB_USE_ABSPATH=False,
     BUILD_DIR=fw_build_meta["build_dir"],
     IS_BASE_FIRMWARE=fw_build_meta["type"] == "firmware",
@@ -76,7 +80,6 @@ if not env["VERBOSE"]:
         HEXCOMSTR="\tHEX\t${TARGET}",
         BINCOMSTR="\tBIN\t${TARGET}",
         DFUCOMSTR="\tDFU\t${TARGET}",
-        OPENOCDCOMSTR="\tFLASH\t${SOURCE}",
     )
 
 
@@ -139,7 +142,7 @@ apps_c = fwenv.ApplicationsC(
     Value(fwenv["APPS"]),
 )
 # Adding dependency on manifest files so apps.c is rebuilt when any manifest is changed
-fwenv.Depends(apps_c, fwenv.GlobRecursive("*.fam", "applications"))
+fwenv.Depends(apps_c, fwenv.GlobRecursive("*.fam", "#/applications"))
 
 sources = [apps_c]
 # Gather sources only from app folders in current configuration
@@ -164,6 +167,8 @@ fwenv.AppendUnique(
         "-u",
         "_printf_float",
         "-n",
+        "-Xlinker",
+        "-Map=${TARGET}.map",
     ],
 )
 
@@ -202,7 +207,6 @@ fwelf = fwenv["FW_ELF"] = fwenv.Program(
     ],
 )
 
-# Make it depend on everything child builders returned
 # Firmware depends on everything child builders returned
 Depends(fwelf, lib_targets)
 # Output extra details after building firmware
@@ -232,7 +236,8 @@ if should_gen_cdb_and_link_dir(fwenv, BUILD_TARGETS):
     fwcdb = fwenv.CompilationDatabase()
     # without filtering, both updater & firmware commands would be generated
     fwenv.Replace(COMPILATIONDB_PATH_FILTER=fwenv.subst("*${FW_FLAVOR}*"))
-    Depends(fwcdb, fwelf)
+    AlwaysBuild(fwcdb)
+    Alias(fwenv["FIRMWARE_BUILD_CFG"] + "_cdb", fwcdb)
     fw_artifacts.append(fwcdb)
 
     # Adding as a phony target, so folder link is updated even if elf didn't change

firmware/targets/f7/stm32wb55xx_flash.ld

@@ -75,6 +75,7 @@ SECTIONS
   .text :
   {
     . = ALIGN(4);
+    *lib*.a:*(.text .text.*) /* code from libraries before apps */
     *(.text)           /* .text sections (code) */
     *(.text*)          /* .text* sections (code) */
     *(.glue_7)         /* glue arm to thumb code */