diff --git a/Sming/Libraries/OtaUpgradeMqtt/README.rst b/Sming/Libraries/OtaUpgradeMqtt/README.rst index 56ff2e7808..3239e27b14 100644 --- a/Sming/Libraries/OtaUpgradeMqtt/README.rst +++ b/Sming/Libraries/OtaUpgradeMqtt/README.rst @@ -1,7 +1,7 @@ OTA Firmware Upgrade via MQTT ============================= -.. highlight:: bash +.. highlight:: c++ Introduction ------------ @@ -9,6 +9,55 @@ Introduction This library allows Sming applications to upgrade their firmware Over-The-Air (OTA) using the MQTT protocol. MTQTT has less overhead compared to HTTP and can be used for faster delivery of application updates. +Using +----- + +1. Add ``COMPONENT_DEPENDS += OtaUpgradeMqtt`` to your application componenent.mk file. +2. Add these lines to your application:: + + #include + + #if ENABLE_OTA_ADVANCED + #include + #endif + + MqttClient mqtt; + + // Call when IP address has been obtained + void onIp(IpAddress ip, IpAddress mask, IpAddress gateway) + { + // ... + + mqtt.connect(Url(MQTT_URL), "sming"); + + #if ENABLE_OTA_ADVANCED + /* + * The advanced parser suppors all firmware upgrades supported by the `OtaUpgrade` library. + * `OtaUpgrade` library provides firmware signing, firmware encryption and so on. + */ + auto parser = new OtaUpgrade::Mqtt::AdvancedPayloadParser(APP_VERSION_PATCH); + #else + /* + * The command below uses class that stores the firmware directly + * using RbootOutputStream on a location provided by us + */ + auto parser = new OtaUpgrade::Mqtt::RbootPayloadParser(part, APP_VERSION_PATCH); + #endif + + mqtt.setPayloadParser([parser] + (MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, int length) -> int + { + return parser->parse(state, message, buffer, length); + }); + + String updateTopic = "/app/test/u/4.3"; + mqtt.subscribe(updateTopic); + + // ... + } + +See the :sample:`Upgrade` sample application. + Versioning Principles --------------------- To simplify the OTA process we strongly recommend the following versioning principles for your application: @@ -29,11 +78,11 @@ Theory Of Operation Depending on the size of the new firmware and the speed of the connection an update can take 10 to 20 seconds. 2. The application connects via MQTT to a remote server and subscribes to a special topic. The topic is based on the - application id and its current version. If the current application version is 4.3.1 then the topic that will be used for OTA is ``/a/test/u/4.3``. + application id and its current version. If the current application id is ``test`` and version is ``4.3.1`` then the topic that will be used for OTA is ``a/test/u/4.3``. 3. If there is a need to support both stable and unstable/nightly builds then the topic name can have `s` or `u` suffix. For example - all stable versions should be published and downloaded from the topic ``/a/test/u/4.3/s``. For the unstable ones we can use the topic ``/a/test/u/4.3/u``. - If an application is interested in both then it can subscribe using the following pattern ``/a/test/u/4.3/+``. + all stable versions should be published and downloaded from the topic ``a/test/u/4.3/s``. For the unstable ones we can use the topic ``a/test/u/4.3/u``. + If an application is interested in both stable and unstable versions then it can subscribe using the following pattern ``a/test/u/4.3/+``. 4. The application is waiting for new firmware. When the application is on battery than it makes sense to wait for a limited time and if there is no message coming back to disconnect. @@ -51,6 +100,27 @@ One MQTT message contains: Based on the :envvar:`ENABLE_OTA_VARINT_VERSION` the patch version can be encoded either using one byte or a `varint `_. Based on :envvar:`ENABLE_OTA_ADVANCED` the firmware data can be either without any encoding or be signed and encrypted. +To simplify the packaging this library comes with a tool called ``deployer``. To create a package type the following from your application:: + + make ota-pack OTA_PATCH_VERSION=127 + +Replace 127 with the desired patch version. +If the option ``OTA_PATCH_VERSION`` is omitted from the command line then the patch version will be generated automatically and it will contain the current unix timestamp. + +Once a package is created it can be deployed to the firmware MQTT server using the command below:: + + make ota-deploy MQTT_FIRMWARE_URL=mqtt://relser:relpassword@attachix.com/a/test/4.3 + +The ``MQTT_FIRMWARE_URL`` above specifies that + +- protocol is: mqtt without SSL. +- user is: relser +- password is: relpassword +- host is: attachix.com +- path is: /a/test/4.3. The topic name is based on the path with removed leading and ending slashes. + +Make sure to replace the MQTT_FIRMWARE_URL value with your MQTT server credentials, host and topic. + Security -------- For additional security a standard SSL/TLS can be used diff --git a/Sming/Libraries/OtaUpgradeMqtt/api.rst b/Sming/Libraries/OtaUpgradeMqtt/api.rst index 37505c0733..e9aca95651 100644 --- a/Sming/Libraries/OtaUpgradeMqtt/api.rst +++ b/Sming/Libraries/OtaUpgradeMqtt/api.rst @@ -5,3 +5,4 @@ OTA Upgrade over MQTT classes .. doxygenclass:: OtaUpgrade::Mqtt::PayloadParser .. doxygenclass:: OtaUpgrade::Mqtt::RbootPayloadParser +.. doxygenclass:: OtaUpgrade::Mqtt::AdvancedPayloadParser diff --git a/Sming/Libraries/OtaUpgradeMqtt/component.mk b/Sming/Libraries/OtaUpgradeMqtt/component.mk index 8583388890..4c78279782 100644 --- a/Sming/Libraries/OtaUpgradeMqtt/component.mk +++ b/Sming/Libraries/OtaUpgradeMqtt/component.mk @@ -16,4 +16,34 @@ COMPONENT_VARS += ENABLE_OTA_VARINT_VERSION ENABLE_OTA_VARINT_VERSION ?= 1 COMPONENT_CXXFLAGS := -DENABLE_OTA_ADVANCED=$(ENABLE_OTA_ADVANCED) \ - -DENABLE_OTA_VARINT_VERSION=$(ENABLE_OTA_VARINT_VERSION) \ No newline at end of file + -DENABLE_OTA_VARINT_VERSION=$(ENABLE_OTA_VARINT_VERSION) + +##@Firmware Upgrade + +OTA_TOOLS := $(COMPONENT_PATH)/tools +OTA_DEPLOYMENT_TOOL = $(OTA_TOOLS)/deployer/out/Host/debug/firmware/deployer$(TOOL_EXT) + +$(OTA_DEPLOYMENT_TOOL): + $(Q) $(MAKE) -C $(OTA_TOOLS)/deployer SMING_ARCH=Host ENABLE_CUSTOM_LWIP=2 + + +# SDP = Sming Deployment Pakage +OTA_PACKAGE_EXT =.sdp + +OTA_PATCH_VERSION ?= $(shell date +%s) + +PACKAGE_IN = $(RBOOT_ROM_0_BIN) +ifneq ($(ENABLE_OTA_ADVANCED), 0) + PACKAGE_IN = $(OTA_UPGRADE_FILE) +endif +PACKAGE_OUT = $(PACKAGE_IN)$(OTA_PACKAGE_EXT) + +.PHONY: ota-pack +ota-pack: $(PACKAGE_OUT) ##Creates a deployment package from the current application (use OTA_PATCH_VERSION to specify the version number) + +$(PACKAGE_OUT): $(PACKAGE_IN) $(OTA_DEPLOYMENT_TOOL) + $(Q) $(OTA_DEPLOYMENT_TOOL) pack -- $(PACKAGE_IN) $(PACKAGE_OUT) $(OTA_PATCH_VERSION) $(ENABLE_OTA_VARINT_VERSION) 2>/dev/null + +.PHONY: ota-deploy +ota-deploy: $(PACKAGE_OUT) ##Uploads new firmware version of the current application (use MQTT_FIRMWARE_URL to specify the MQTT URL) + $(Q) $(OTA_DEPLOYMENT_TOOL) deploy -- $(PACKAGE_OUT) $(MQTT_FIRMWARE_URL) 2>/dev/null diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst index ff41d9865e..cc14bab324 100644 --- a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/README.rst @@ -11,6 +11,12 @@ This application uses :component:`OtaUpgradeMqtt` and follows the recommended ve Based on :envvar:`ENABLE_OTA_ADVANCED` the firmware data can be either without any encoding or be signed and encrypted. +Tools +----- +There are two tools that facilitate the packiging and deployment of a new firmware. + +For more information read ``Firmware packaging`` in the documentation of the :component:`OtaUpgradeMqtt` component. + Security -------- Depending on :envvar:`ENABLE_SSL` a standard SSL/TLS can be enabled. This way diff --git a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp index 708f7c4c10..87dbb20d31 100644 --- a/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp +++ b/Sming/Libraries/OtaUpgradeMqtt/samples/Upgrade/app/application.cpp @@ -98,7 +98,7 @@ void otaUpdate() mqtt.setPayloadParser([parser](MqttPayloadParserState& state, mqtt_message_t* message, const char* buffer, int length) -> int { return parser->parse(state, message, buffer, length); }); - String updateTopic = "/a/"; + String updateTopic = "a/"; updateTopic += APP_ID; updateTopic += "/u/"; updateTopic += APP_VERSION; diff --git a/Sming/Libraries/OtaUpgradeMqtt/src/.cs b/Sming/Libraries/OtaUpgradeMqtt/src/.cs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject new file mode 100644 index 0000000000..fecee42f2f --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.cproject @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + make + + all + true + true + true + + + make + + rebuild + true + true + true + + + make + + flash + true + true + true + + + + diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project new file mode 100644 index 0000000000..0b94d5c588 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/.project @@ -0,0 +1,30 @@ + + + FirmwareDeployer + + + SmingFramework + Libraries + Sming + + + + org.eclipse.cdt.managedbuilder.core.genmakebuilder + clean,full,incremental, + + + + + org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder + full,incremental, + + + + + + org.eclipse.cdt.core.cnature + org.eclipse.cdt.core.ccnature + org.eclipse.cdt.managedbuilder.core.managedBuildNature + org.eclipse.cdt.managedbuilder.core.ScannerConfigNature + + diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile new file mode 100644 index 0000000000..ff51b6c3a7 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/Makefile @@ -0,0 +1,9 @@ +##################################################################### +#### Please don't change this file. Use component.mk instead #### +##################################################################### + +ifndef SMING_HOME +$(error SMING_HOME is not set: please configure it as an environment variable) +endif + +include $(SMING_HOME)/project.mk diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst new file mode 100644 index 0000000000..97312c4c02 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/README.rst @@ -0,0 +1,62 @@ +Device_Scanner +============== + +.. highlight:: bash + +Utility to package and deploy firmware files for upgrade over MQTT +See :library:`OtaUpgradeMqtt` for details. + + +Packaging +--------- + +To scan your local network, do this:: + + make pack HOST_PARAMETERS=scan + + +This will start by scanning for all root devices ``:upnp:rootdevice``. +If you want to use a specific search type, just add it to the command line:: + + make run HOST_PARAMETERS='scan urn:schemas-upnp-org:service:ContentDirectory:1' + +This will take some time to complete. If there are any failures, you can retry +by running the command again: any descriptions which have already been fetched will +be skipped. + +You find output in the following directories: + +out/upnp/devices + Contains a hierarchical map of all found devices on your local UPnP network. + + +The schema directories are populated with modified versions of the description files. +These reflect the C++ class structures which we will later create: + +out/upnp/schema/services + All services are extracted into a directory by ``{domain}``. Each service has a separate .xml file. + + Service schema are re-usable and new ones should be added to the UPnP library so they are available for + everyone to use. + + +out/upnp/schema/devices + All devices (both root and embedded) are extracted to a separate directory ``{manufacturer}/{friendlyName}`` + with a single .xml file for each device. + + Device schema are specific to each device implementation so will generally be kept in a separate library + or application. + + +Processing existing descriptions +-------------------------------- + +You can also process device descriptions files pulled in via other means. For example: + + make run HOST_PARAMETERS='parse config/panasonic/viera dmr/ddd.xml dms/ddd.xml nrc/ddd.xml' + +The first parameter ``parse`` is the command. +The second parameter ``config/panasonic/viera`` gives the root directory for the device. +The remaining parameters are the relative locations from this directory of a device description file. +References to service files are pulled in: if they are missing, this may fail. + diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp new file mode 100644 index 0000000000..a436c2dbe2 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/app/application.cpp @@ -0,0 +1,210 @@ +#include +#include + +#ifdef ARCH_HOST +#ifdef __WIN32__ +#include +#endif + +#include +#include +#include +#endif + +// If you want, you can define WiFi settings globally in Eclipse Environment Variables +#ifndef WIFI_SSID +#define WIFI_SSID "PleaseEnterSSID" // Put you SSID and Password here +#define WIFI_PWD "PleaseEnterPass" +#endif + +namespace +{ +MqttClient mqtt; + +#ifdef ARCH_HOST + +size_t charsWriter(const char* buffer, size_t length) +{ + return fwrite(buffer, sizeof(char), length, stdout); +} + +template void print(const T& arg) +{ + String s(arg); + m_nputs(s.c_str(), s.length()); +} + +void println() +{ + m_puts("\r\n"); +} + +template void println(const T& arg) +{ + print(arg); + println(); +} + +int writePatchVersion(int patchVersion, bool useVarInt, ReadWriteStream* output) +{ + if(output == nullptr) { + return -1; + } + + int written = 0; + if(useVarInt) { + while(patchVersion > 0x7f) { + if(output->write(((uint8_t)(patchVersion)) | 0x80) < 0) { + return false; + } + patchVersion >>= 7; + written++; + } + if(output->write(((uint8_t)patchVersion) & 0x7f) < 0) { + return false; + } + written++; + } else { + written = output->write((uint8_t)patchVersion); + } + + return written; +} + +bool pack(const String& inputFileName, const String& outputFileName, size_t patchVersion, bool useVarInt) +{ + HostFileStream input; + input.open(inputFileName); + if(!input.fileExist()) { + m_printf(_F("ERROR: Invalid input file: %s\r\n"), inputFileName.c_str()); + return false; + } + + HostFileStream output; + output.open(outputFileName, eFO_CreateNewAlways | eFO_WriteOnly); + writePatchVersion(patchVersion, useVarInt, &output); + output.copyFrom(&input); + output.close(); + + return true; +} + +bool deploy(const String& outputFileName, const String& url) +{ + if(mqtt.isProcessing()) { + // we are still processing the data... + return false; + } + + HostFileStream* output = new HostFileStream(); + output->open(outputFileName); + if(!output->fileExist()) { + m_printf(_F("ERROR: Invalid input file: %s"), outputFileName.c_str()); + return false; + } + + WifiStation.enable(true, false); + WifiStation.config(WIFI_SSID, WIFI_PWD); + WifiAccessPoint.enable(false, false); + WifiEvents.onStationGotIP([url, output](IpAddress ip, IpAddress netmask, IpAddress gateway) { + Url mqttUrl(url); + + mqtt.connect(mqttUrl, "sming"); + mqtt.setConnectedHandler([mqttUrl, output](MqttClient& client, mqtt_message_t* message) -> int { + if(message == nullptr) { + // invalid message received + return 1; + } + + if(message->connack.return_code) { + m_printf(_F("ERROR: Connection failed. Reason: %s"), + mqtt_connect_error_string(static_cast(message->connack.return_code))); + + return 0; + } + + uint8_t retained = 1; + uint8_t QoS = 2; + uint8_t flags = (uint8_t)(retained + (QoS << 1)); + mqtt.publish(mqttUrl.Path.substring(1), output, flags); + mqtt.setPublishedHandler([](MqttClient& client, mqtt_message_t* message) -> int { + println(F("Firmware uploaded successfully.")); + System.restart(); + + return 0; + }); + + return 0; + }); + }); + + return true; +} + +void help() +{ + println(); + println(F("Available commands:")); + println(F(" pack fileName.in fileName.out patchVersion Creates a package to be deployed on " + "firmware upgrade server.")); + println(F(" deploy fileName.out mqttUrl Deploys a deployment package to " + "firmware upgrade server.")); + println(); +} + +/* + * Return true to continue execution, false to quit. + */ +bool parseCommands() +{ + auto parameters = commandLine.getParameters(); + if(parameters.count() == 0) { + help(); + return false; + } + + String cmd = parameters[0].text; + if(cmd == "pack") { + if(parameters.count() >= 4) { + bool useVarInt = false; + if(parameters.count() > 4) { + useVarInt = strtol(parameters[4].text, nullptr, 0); + } + if(pack(parameters[1].text, parameters[2].text, strtol(parameters[3].text, nullptr, 0), useVarInt)) { + } + } + + return false; // after packaging the application can be terminated + } + + if(cmd == "deploy") { + if(parameters.count() < 3) { + m_printf(_F("ERROR: Specify filename.out and MQTT_FIRMWARE_URL.\r\n")); + return false; + } + + return deploy(parameters[1].text, parameters[2].text); + } + + help(); + return false; +} + +#endif // ARCH_HOST + +} // namespace + +void init() +{ + Serial.setTxBufferSize(1024); + Serial.begin(SERIAL_BAUD_RATE); + Serial.systemDebugOutput(true); + +#ifdef ARCH_HOST + m_setPuts(charsWriter); + + if(!parseCommands()) { + System.restart(1000); + } +#endif +} diff --git a/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk new file mode 100644 index 0000000000..fc77dcce33 --- /dev/null +++ b/Sming/Libraries/OtaUpgradeMqtt/tools/deployer/component.mk @@ -0,0 +1,14 @@ +COMPONENT_DEPENDS := OtaUpgradeMqtt +APP_NAME := deployer + +SMING_ARCH := Host + +##@Firmware Upgrade + +APP=$(TARGET_OUT_0) + +pack: application $(APP) ##Pack firmware files for deployment (HOST_PARAMETERS=inputFile packedFile version 1|0. Example: ) + $(Q) $(APP) pack -- $(HOST_PARAMETERS) + +deploy: application $(APP) ##Deploy firmware files to MQTT server (HOST_PARAMETERS=packedFile MQTTQ_URL. Example MQTT user: ) + $(Q) $(APP) deploy -- $(HOST_PARAMETERS) \ No newline at end of file