[OTA] Enable 2 clicks OTA update / one simple MQTT command (#1485)

* [OTA] Add automatic OTA updates

This feature will enable to update the gateway with one button or a simple MQTT command.

* [CI] Enable automatic OTA for nightly development builds

And integrate the latest_version file creation into the CI

Enable to specify a particular version
and switch between dev and production builds

.github/workflows/build_and_docs_to_dev.yml

@@ -25,11 +25,13 @@ jobs:
           npm install
       - name: Set sha tag
         run: |
-          sed -i "s/version_tag/${{ steps.short-sha.outputs.sha }}/g" main/User_config.h
+          sed -i "s/version_tag/${{ steps.short-sha.outputs.sha }}/g" main/User_config.h scripts/latest_version_dev.json
           sed -i "s/version_tag/DEVELOPMENT SHA:${{ steps.short-sha.outputs.sha }} TEST ONLY/g" docs/.vuepress/config.js
           sed -i "s|base: '/'|base: '/dev/'|g" docs/.vuepress/config.js
       - name: Run PlatformIO
-        run: platformio run
+        run: |
+          export PLATFORMIO_BUILD_FLAGS="'-DDEVELOPMENTOTA=true'"
+          platformio run
       - name: Prepare Release Assets
         run: |
           sudo apt install rename

.github/workflows/release.yml

@@ -19,7 +19,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install platformio
       - name: Set version tag from git
-        run: sed -i "s/version_tag/${GITHUB_REF#refs/tags/}/g" main/User_config.h
+        run: sed -i "s/version_tag/${GITHUB_REF#refs/tags/}/g" main/User_config.h scripts/latest_version.json
       - name: Run PlatformIO
         run: platformio run
       - name: Prepare Release Assets

docs/use/gateway.md

@@ -104,20 +104,29 @@ The `mqtt_cert_index` value corresponds to the 0 to X index of the `certs_array`
 
 # Firmware update from MQTT (ESP only)
 
-The gateway can be updated through an MQTT message by providing a JSON formatted message with a version number, OTA password (optional, see below), and URL to fetch the update from.  
+When the gateway used is from a standard ESP32 environment [listed and defined here](https://github.com/1technophile/OpenMQTTGateway/blob/development/environments.ini), it can be updated through a simple MQTT command:
+```
+mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/MQTTtoSYS/firmware_update" -m '{
+  "version": "latest"
+}'
+```
+This would download the latest version firmware binary from Github and install it.
 
-To enable this functionality, `MQTT_HTTPS_FW_UPDATE` will need to be defined or the line that defines in in user_config.h will need to be uncommented.
+Note that this update option is also autodiscovered through Home Assistant convention, you can update directly from the device page with 2 clicks.
 
-::: tip
-If using an unsecure MQTT broker it is **highly recommended** to disable the password checking by setting the macro `MQTT_HTTPS_FW_UPDATE_USE_PASSWORD` to 0 (default is 1 (enabled)), otherwise a clear text password may be sent over the network.  
+![](../img/OpenMQTTGateway-OTA-Update-Home-Assistant.png)
 
-The `server_cert` parameter is optional. If the update server has changed or certificate updated or not set in `user_config.h` then you can provide the certificate here.
-:::
+You can also indicate the target version to update:
+```
+mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/MQTTtoSYS/firmware_update" -m '{
+  "version": "v1.2.0"
+}'
+```
+Alternatively if you want to choose the update URL you can use the command below (ESP32 and ESP8266):
 
-### Example firmware update message:
 Without certificate, in this case we will use the root certificate defined in User_config.h
 ```
-mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/firmware_update" -m '{
+mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/MQTTtoSYS/firmware_update" -m '{
   "version": "test",
   "password": "OTAPASSWORD",
   "url": "https://github.com/1technophile/OpenMQTTGateway/releases/download/v0.9.12/esp32dev-ble-firmware.bin"
@@ -126,7 +135,7 @@ mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/firmware_update" -m '{
 
 With certificate:
 ```
-mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/firmware_update" -m '{
+mosquitto_pub -t "home/OpenMQTTGateway_ESP32_BLE/commands/MQTTtoSYS/firmware_update" -m '{
   "version": "test",
   "password": "OTAPASSWORD",
   "url": "https://github.com/1technophile/OpenMQTTGateway/releases/download/v0.9.12/esp32dev-ble-firmware.bin",
@@ -156,6 +165,15 @@ CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
 
 A bash script is available [here](ota_command_cert.zip) to simplify the use of the `server_cert` parameter.  
 
+
+To enable this functionality, `MQTT_HTTPS_FW_UPDATE` will need to be defined or the line that defines in in user_config.h will need to be uncommented.
+
+::: tip
+If using an unsecure MQTT broker it is **highly recommended** to disable the password checking by setting the macro `MQTT_HTTPS_FW_UPDATE_USE_PASSWORD` to 0 (default is 1 (enabled)), otherwise a clear text password may be sent over the network.  
+
+The `server_cert` parameter is optional. If the update server has changed or certificate updated or not set in `user_config.h` then you can provide the certificate here.
+:::
+
 ::: warning
 The pre-built binaries for **rfbridge** and **avatto-bakeey-ir** have the above WiFi and MQTT broker credentials and the Firmware update via MQTT options disabled. This is due to the restricted available flash, so as to still be able to use OTA firmware updates for these boards.
 :::

main/User_config.h

@@ -238,6 +238,14 @@ CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
 #    ifndef MQTT_HTTPS_FW_UPDATE_USE_PASSWORD
 #      define MQTT_HTTPS_FW_UPDATE_USE_PASSWORD 1 // Set this to 0 if not using TLS connection to MQTT broker to prevent clear text passwords being sent.
 #    endif
+#    if DEVELOPMENTOTA
+#      define OTA_JSON_URL "https://github.com/1technophile/OpenMQTTGateway/raw/gh-pages/dev/firmware_build/latest_version_dev.json" //OTA url used to discover new versions of the firmware from development nightly builds
+#    else
+#      define OTA_JSON_URL "https://github.com/1technophile/OpenMQTTGateway/raw/gh-pages/firmware_build/latest_version.json" //OTA url used to discover new versions of the firmware
+#    endif
+#    define ENTITY_PICTURE   "https://github.com/1technophile/OpenMQTTGateway/raw/development/docs/img/Openmqttgateway_logo_mini_margins.png"
+#    define RELEASE_LINK_DEV "https://github.com/1technophile/OpenMQTTGateway/raw/gh-pages/dev/firmware_build/"
+#    define RELEASE_LINK     "https://github.com/1technophile/OpenMQTTGateway/releases/download/"
 #  endif
 
 #  ifndef MQTT_SECURE_SELF_SIGNED
@@ -558,7 +566,6 @@ CRGB leds2[FASTLED_IND_NUM_LEDS];
 #define subjectMQTTtoX     "/commands/#"
 #define subjectMultiGTWKey "toMQTT"
 #define subjectGTWSendKey  "MQTTto"
-#define subjectFWUpdate    "firmware_update"
 
 // key used for launching commands to the gateway
 #define restartCmd "restart"
@@ -599,6 +606,7 @@ CRGB leds2[FASTLED_IND_NUM_LEDS];
 #define InitialMQTTConnectionTimeout 10 // time estimated (s) before the board is connected to MQTT
 #define subjectSYStoMQTT             "/SYStoMQTT"
 #define subjectMQTTtoSYSset          "/commands/MQTTtoSYS/config"
+#define subjectMQTTtoSYSupdate       "/commands/MQTTtoSYS/firmware_update"
 #define TimeToResetAtStart           5000 // Time we allow the user at start for the reset command by button press
 /*-------------------DEFINE LOG LEVEL----------------------*/
 #ifndef LOG_LEVEL

main/ZmqttDiscovery.ino

@@ -287,8 +287,10 @@ void createDiscovery(const char* sensor_type,
       sensor["pl_prs"] = payload_on; // payload_press for a button press
     } else if (strcmp(sensor_type, "number") == 0) {
       sensor["cmd_tpl"] = payload_on; // payload_on for a switch
+    } else if (strcmp(sensor_type, "update") == 0) {
+      sensor["payload_install"] = payload_on; // payload_install for update
     } else {
-      sensor["pl_on"] = payload_on; // payload_on for a switch
+      sensor["pl_on"] = payload_on; // payload_on for the rest
     }
   }
   if (payload_off[0])
@@ -530,6 +532,15 @@ void pubMqttDiscovery() {
                   "", "", "", "", false, // device name, device manufacturer, device model, device ID, retain
                   stateClassNone //State Class
   );
+  createDiscovery("update", //set Type
+                  subjectSYStoMQTT, "SYS: Firmware Update", (char*)getUniqueId("update", "").c_str(), //set state_topic,name,uniqueId
+                  will_Topic, "firmware", "", //set availability_topic,device_class,value_template,
+                  LATEST_OR_DEV, "", "", //set,payload_on,payload_off,unit_of_meas,
+                  0, //set  off_delay
+                  Gateway_AnnouncementMsg, will_Message, true, subjectMQTTtoSYSupdate, //set,payload_available,payload_not available   ,is a gateway entity, command topic
+                  "", "", "", "", false, // device name, device manufacturer, device model, device ID, retain
+                  stateClassNone //State Class
+  );
 #  ifdef ZsensorBME280
 #    define BMEparametersCount 5
   Log.trace(F("bme280Discovery" CR));

main/config_mqttDiscovery.h

@@ -268,4 +268,10 @@ const char* availableHASSUnits[] = {"W",
                                     "mm/h",
                                     "cm"};
 
+// Define the command used to update through OTA depending if we want to update from dev nightly or latest release
+#if DEVELOPMENTOTA
+#  define LATEST_OR_DEV "{\"version\":\"dev\"}"
+#else
+#  define LATEST_OR_DEV "{\"version\":\"latest\"}"
+#endif
 #endif

main/main.ino

@@ -596,9 +596,6 @@ void connectMQTT() {
 #endif
 #ifdef ZgatewayIR
       client.subscribe(subjectMultiGTWIR); // subject on which other OMG will publish, this OMG will store these msg and by the way don't republish them if they have been already published
-#endif
-#ifdef MQTT_HTTPS_FW_UPDATE
-      client.subscribe(subjectFWUpdate);
 #endif
       Log.trace(F("Subscription OK to the subjects %s" CR), topic2);
     }
@@ -653,7 +650,7 @@ void callback(char* topic, byte* payload, unsigned int length) {
   //launch the function to treat received data if this data concern OpenMQTTGateway
   if ((strstr(topic, subjectMultiGTWKey) != NULL) ||
       (strstr(topic, subjectGTWSendKey) != NULL) ||
-      (strstr(topic, subjectFWUpdate) != NULL))
+      (strstr(topic, subjectMQTTtoSYSupdate) != NULL))
     receivingMQTT(topic, (char*)p);
 
   // Free the memory
@@ -1509,6 +1506,10 @@ void loop() {
       connected = true;
 #if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
       if (now > (timer_sys_measures + (TimeBetweenReadingSYS * 1000)) || !timer_sys_measures) {
+#  if defined(ESP32) && defined(MQTT_HTTPS_FW_UPDATE)
+        if (!timer_sys_measures) // Only check for updates at start for ESP32
+          checkForUpdates();
+#  endif
         timer_sys_measures = millis();
         stateMeasures();
 #  ifdef ZgatewayBT
@@ -1691,6 +1692,7 @@ void stateMeasures() {
   SYSdata["uptime"] = uptime();
   SYSdata["version"] = OMG_VERSION;
 #  if defined(ESP8266) || defined(ESP32)
+  SYSdata["env"] = ENV_NAME;
   uint32_t freeMem;
   freeMem = ESP.getFreeHeap();
   SYSdata["freemem"] = freeMem;
@@ -1940,38 +1942,96 @@ void receivingMQTT(char* topicOri, char* datacallback) {
 }
 
 #ifdef MQTT_HTTPS_FW_UPDATE
+String latestVersion;
 #  ifdef ESP32
+#    include <HTTPClient.h>
+
 #    include "zzHTTPUpdate.h"
+
+/**
+ * Check on a server the latest version information to build a releaseLink
+ * The release link will be used when the user trigger an OTA update command
+ * Only available for ESP32
+ */
+bool checkForUpdates() {
+  HTTPClient http;
+  http.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
+  http.begin(OTA_JSON_URL, OTAserver_cert);
+  int httpCode = http.GET();
+  StaticJsonDocument<JSON_MSG_BUFFER> jsonBuffer;
+  JsonObject jsondata = jsonBuffer.to<JsonObject>();
+
+  if (httpCode > 0) { //Check for the returning code
+    String payload = http.getString();
+    auto error = deserializeJson(jsonBuffer, payload);
+    if (error) {
+      Log.error(F("Deserialize MQTT data failed: %s" CR), error.c_str());
+    }
+    Log.trace(F("HttpCode %d" CR), httpCode);
+    Log.trace(F("Payload %s" CR), payload.c_str());
+  } else {
+    Log.error(F("Error on HTTP request"));
+  }
+  http.end(); //Free the resources
+  if (jsondata.containsKey("latest_version")) {
+    jsondata["installed_version"] = OMG_VERSION;
+    jsondata["entity_picture"] = ENTITY_PICTURE;
+    latestVersion = jsondata["latest_version"].as<String>();
+    pub(subjectSYStoMQTT, jsondata);
+    Log.trace(F("Update file found on server" CR));
+    return true;
+  } else {
+    Log.trace(F("No update file found on server" CR));
+    return false;
+  }
+}
 #  elif ESP8266
 #    include <ESP8266httpUpdate.h>
 #  endif
+
 void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
-  if (strstr(topicOri, subjectFWUpdate) != NULL) {
+  if (strstr(topicOri, subjectMQTTtoSYSupdate) != NULL) {
     const char* version = HttpsFwUpdateData["version"];
     if (version && ((strlen(version) != strlen(OMG_VERSION)) || strcmp(version, OMG_VERSION) != 0)) {
       const char* url = HttpsFwUpdateData["url"];
+      String systemUrl;
       if (url) {
         if (!strstr((url + (strlen(url) - 5)), ".bin")) {
           Log.error(F("Invalid firmware extension" CR));
           return;
         }
-      } else {
-        Log.error(F("Invalid URL" CR));
-        return;
-      }
-
 #  if MQTT_HTTPS_FW_UPDATE_USE_PASSWORD > 0
-      const char* pwd = HttpsFwUpdateData["password"];
-      if (pwd) {
-        if (strcmp(pwd, ota_pass) != 0) {
-          Log.error(F("Invalid OTA password" CR));
+        const char* pwd = HttpsFwUpdateData["password"];
+        if (pwd) {
+          if (strcmp(pwd, ota_pass) != 0) {
+            Log.error(F("Invalid OTA password" CR));
+            return;
+          }
+        } else {
+          Log.error(F("No password sent" CR));
           return;
         }
+#  endif
+#  ifdef ESP32
+      } else if (strcmp(version, "latest") == 0) {
+        if (!checkForUpdates())
+          return;
+        systemUrl = RELEASE_LINK + latestVersion + "/" + ENV_NAME + "-firmware.bin";
+        url = systemUrl.c_str();
+        Log.notice(F("Using system OTA url with latest version %s" CR), url);
+      } else if (strcmp(version, "dev") == 0) {
+        systemUrl = String(RELEASE_LINK_DEV) + ENV_NAME + "-firmware.bin";
+        url = systemUrl.c_str();
+        Log.notice(F("Using system OTA url with dev version %s" CR), url);
+      } else if (version[0] == 'v') {
+        systemUrl = String(RELEASE_LINK) + version + "/" + ENV_NAME + "-firmware.bin";
+        url = systemUrl.c_str();
+        Log.notice(F("Using system OTA url with defined version %s" CR), url);
+#  endif
       } else {
-        Log.error(F("No password sent" CR));
+        Log.error(F("Invalid URL" CR));
         return;
       }
-#  endif
 
       Log.warning(F("Starting firmware update" CR));
       SendReceiveIndicatorON();

platformio.ini

@@ -177,6 +177,7 @@ lib_deps =
   ${env.lib_deps}
 build_flags =
   ${env.build_flags}
+  '-DENV_NAME="$PIOENV"'
   '-DZmqttDiscovery="HADiscovery"'
   '-DTRACE=1'
   '-DCONFIG_BT_NIMBLE_ROLE_PERIPHERAL_DISABLED'