MQTT Changes Topic Name to "Errors" After Hard Fault

[udellc]

Describe the bug
WeatherChimes publishes data to MongoDB via MQTT. The current method automatically names the MQTT topic from the sending device's name and id number from the manager. This works fine until there is a Hard Fault in the system; in which case, the topic name being published is changed to "Errors[device id number]."
SD publishing seems to handle hard fault cases the way we want, continuing to log to the same file as intended, and only logging errors to the errors.csv file.

I suspect, because the hard fault is occurring in the SD.cpp area, that the current method of building the MQTT JSON document after the faulting log SD operation is screwing up the name in the JSON area. As a band aid we could move MQTT logging to before the SD log operation, but it would be even better to fix how we are generating the MQTT JSON document so even if we get a hard fault we will still publish data to the intended topic.

Chime #4 (Chime #3 seems to be experiencing additional SDI-12 related hardware issues)
Feather M0 WiFi
Hypnos Board: SD, DS3231 RTC, 3v 5v Power rail switching
TSL2591
GS3 (SDI-12 soil moisture sensor by Meter)
SHT-30 Air Temp, Humidity

To Reproduce
Steps to reproduce the behavior:
Change WiFi router info in arduino_secrets for: SECRET_SSID, SECRET_PASS for your internet router
Ask Will or Chet for arduino secrets for MQTT broker and input
Upload attached WeatherChimesV4SDSleepMQTT.ino and config.h to WeatherChimes Hardware

Expected behavior
We want the device to continue publishing to the intended MQTT topic [device name][device id] instead of Errors[device id].
SD publishing seems to handle hard fault cases the way we want, continuing to log to the same file as intended, and only logging errors to the errors.csv file.

Code

///////////////////////////////////////////////////////////////////////////////

// This is a basic example that demonstrates how to log data to an SD card
// using Loom.

///////////////////////////////////////////////////////////////////////////////

#include <Loom.h>
#include "MQTT.h"

// These are default times to sleep between cycles if there is no SD_config.txt file exists on SD card for user-set time
int secs = 20;
int mins = 0;
int hours = 0;
int days = 0;

// Include configuration
const char* json_config =
#include "config.h"
;

// In Tools menu, set:
// Internet  > Disabled
// Sensors   > Enabled
// Radios    > Disabled
// Actuators > Disabled
// Max       > Disabled

using namespace Loom;

Loom::Manager Feather{};

char devName[20];
String topic = "";
String jsonSerialized = "";

DynamicJsonDocument doc(1024);

void setup()
{
  delay(3000); // Wait a bit in case we want to program the device
  pinMode(5, OUTPUT);   // Enable control of 3.3V rail 
  pinMode(6, OUTPUT);   // Enable control of 5V rail 

  //See Above
  digitalWrite(5, LOW); // Enable 3.3V rail
  digitalWrite(6, HIGH);  // Enable 5V rail
  
  Feather.begin_LED();
  Feather.begin_serial(true);
  Feather.parse_config(json_config);
  Feather.print_config();

  // Register an interrupt on the RTC alarm pin
  getInterruptManager(Feather).register_ISR(12, wakeISR_RTC, LOW, ISR_Type::IMMEDIATE);

  // Get the device name and then using the device name, instance number and site name create a topic name to publish to
  Feather.get_device_name(devName);
  topic = String(SITE_NAME) + "/" + String(devName) + String(Feather.get_instance_num());

  enable_wifi();

  LPrintln("\n ** Setup Complete ** ");
}


void loop()
{
  Feather.measure();
  Feather.package();
  Feather.display_data();

  // Log using default filename as provided in configuration
  // in this case, 'datafile.csv'
  getSD(Feather).log();

  // Or log to a specific file (does not change what default file is set to)
  // getSD(Feather)log("specific.csv");

  // Log online to MongoDB via MQTT WiFi
  connect_to_wifi();
  connect_to_broker(1);
  // Build JSON document to publish via MQTT
  doc.clear();
  jsonSerialized = "";
  doc.add(Feather.internal_json(false));
  serializeJson(doc, jsonSerialized);
  publish_mqtt(topic, jsonSerialized);

  // Set RTC Timer for wake up and sample to next time interval
  getInterruptManager(Feather).RTC_alarm_duration(TimeSpan(days,hours,mins,secs));
  getInterruptManager(Feather).reconnect_interrupt(12);

  disconnect_wifi(); // Disable WiFi for power savings

  digitalWrite(5, HIGH); // Disable 3.3V rail
  digitalWrite(6, LOW);  // Disable 5V rail

  // Disable SPI pins/SD chip select to save power
  pinMode(23, INPUT);
  pinMode(24, INPUT);
  pinMode(10, INPUT);  // Needs to be correct pin for SD CS on Hypnos

  Feather.power_down();
  getSleepManager(Feather).sleep(); // Sketch pauses here until RTC alarm

  digitalWrite(5, LOW); // Enable 3.3V rail
  digitalWrite(6, HIGH);  // Enable 5V rail

  pinMode(23, OUTPUT);
  pinMode(24, OUTPUT);
  pinMode(10, OUTPUT); // Needs to be correct pin for SD CS on Hypnos

  // *** SITS Here in Sleep till p12 RTC Alarm Wake Signal ...

  Feather.power_up();
  delay(1000);        // Delay for power 
}

void wakeISR_RTC() {  
  // disable the interrupt
  detachInterrupt(12);
}

Config

"{\
  'general':\
  {\
    'name':'Chime',\
    'instance':3,\
    'interval':5000,\
    'print_verbosity':2\
  },\
  'components':[\
    {\
      'name':'SD',\
      'params':[true, 1000, 10,'chimes', true]\
    },\
    {\
      'name':'InterruptManager',\
      'params':'default'\
    },\
    {\
      'name':'SleepManager',\
      'params':[true,false,1]\
    },\
    {\
      'name':'DS3231',\
      'params':[10, true, true]\
    },\
    {\
      'name':'SDIManager',\
      'params':'default'\
    },\
    {\
      'name':'TSL2591',\
      'params':'default'\
    },\
    {\
      'name':'SHT31D',\
      'params':'default'\
    },\
    {\
      'name':'Analog',\
      'params':[\
        8,\
        12,\
        false,\
        false,\
        false,\
        false,\
        false,\
        false,\
        1,\
        1,\
        1,\
        1,\
        1,\
        1,\
        25.0\
      ]\
    }\
  ]\
}"

MQTT.h

#pragma once
#include <ArduinoMqttClient.h>
#include <WiFi101.h>

#include "arduino_secrets.h"

// Feather M0 Wifi Pins
#define WINC_CS   8
#define WINC_IRQ  7
#define WINC_RST  4
#define WINC_EN   2

char ssid[] = SECRET_SSID;
char pass[] = SECRET_PASS;

const char broker[] = SECRET_BROKER;
int port = BROKER_PORT;

WiFiClient wifiClient;
MqttClient mqttClient(wifiClient);

/* Disconnect from broker and WiFi*/
void disconnect_wifi(){
  mqttClient.stop();
  WiFi.disconnect();
  WiFi.end();
}

/**
 * Connect to the wifi network
 */ 
void connect_to_wifi(){

    // Try to conect to the wifi network
    Serial.print("[MQTT] Attempting to connect to SSID: ");
    Serial.println(ssid);

    // Check if there is a password for the wifi or not
    if(strlen(pass) > 0){
        // Try to connect to the Wifi network until it succeeds 
        while (WiFi.begin(ssid, pass) != WL_CONNECTED) {
            Serial.print("[MQTT] Attempting connection to AP...");
            delay(5000);
        }
    }
    else{
        // Try to connect to the Wifi network until it succeeds 
        while (WiFi.begin(ssid) != WL_CONNECTED) {
            Serial.print("[MQTT] Attempting connection to AP...");
            delay(5000);
        }
    }

    Serial.println("Connected to network!");
}

/*
 * Enable the Wifi but don't connect to any network
 */
void enable_wifi(){
  
  // Set the pins that the WiFi module should
  WiFi.setPins(WINC_CS, WINC_IRQ, WINC_RST, WINC_EN);
  
  // Initialise the Client
  Serial.print(F("\nInit the WiFi module..."));
  
  // Check for the presence of the breakout
  if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("WINC1500 not present");
    
    // don't continue:
    while (true);
  }

  // Set the WiFi chip into the lowest power mode to conserve energy
  WiFi.maxLowPowerMode();
  
  Serial.println("ATWINC OK!");
}

/**
 * Connect to the MQTT broker
 */ 
void connect_to_broker(int keepAliveMins){

    // Set the MQTT username and password
    mqttClient.setUsernamePassword(BROKER_USER, BROKER_PASSWORD);

    // Set keep alive time
    mqttClient.setKeepAliveInterval(1000 * 60 * keepAliveMins);

    Serial.print("[MQTT] Attempting to connect to broker: ");
    Serial.println(broker);

    // Try to connect to the broker
    if (!mqttClient.connect(broker, port)) {
        Serial.print("[MQTT] MQTT connection failed! Error code = ");
        Serial.println(mqttClient.connectError());
    }

    Serial.println("[MQTT] You're connected to the MQTT broker!");
}

/**
 * Publish the data to the correct topic
 */ 
void publish_mqtt(String topic, String data){

    // send message, the Print interface can be used to set the message contents
    mqttClient.poll();
    delay(250);         // ****** Added 6.8.22 Test if delay after poll fixes freezing
    mqttClient.beginMessage(topic);
    mqttClient.print(data);
    mqttClient.endMessage();
}

arduino_secrets.h

// Wifi settings
#define SECRET_SSID "routername" // WiFi Name
#define SECRET_PASS ""  // WiFi Password

// MQTT Settings
#define BROKER_USER ""
#define BROKER_PASSWORD ""
#define SECRET_BROKER ""
#define BROKER_PORT 1883
#define SITE_NAME "WeatherChimes" // The name of the location where these nodes will be placed

Additional context
Add any other context about the problem here.
chimes170.csv
Errors.csv

[udellc]

Update: Changing the order of logging from SD then MQTT - to MQTT then SD did not make a difference. It must be that in FeatherFault, when a fault is detected, it renames the Feather.internal_json to "Errors" temporarily. The funny thing is, SD logging is stall able to log data to the correct file.
Recommendation: we need to look at the log SD functions and see from where the JSON packet is being pulled to ensure data keeps going to the proper named file. Then use that example to ensure MQTT logs to the correct topic.
Stretch goal would be to make MQTT log like SD logs: i.e. Sensor data to the correct topic, and any errors to a separate errors topic with a reasonable name to identify from which device the error log belongs to. See code I tried below:

void loop()
{
  Feather.measure();
  Feather.package();
  Feather.display_data();
// *edit: Swapped the order of publishing to MQTT and then to SD
// Because SD Hard Faults and makes the MQTT json packet change topic name to Errors#
  
  // Log online to MongoDB via MQTT WiFi
  connect_to_wifi();
  connect_to_broker(1);
  // Build JSON document to publish via MQTT
  doc.clear();
  jsonSerialized = "";
  doc.add(Feather.internal_json(false));
  serializeJson(doc, jsonSerialized);
  publish_mqtt(topic, jsonSerialized);

  // Log using default filename as provided in configuration
  // in this case, 'datafile.csv'
  getSD(Feather).log();

  // Or log to a specific file (does not change what default file is set to)
  // getSD(Feather)log("specific.csv");

  // Set RTC Timer for wake up and sample to next time interval
  getInterruptManager(Feather).RTC_alarm_duration(TimeSpan(days,hours,mins,secs));
  getInterruptManager(Feather).reconnect_interrupt(12);

  disconnect_wifi(); // Disable WiFi for power savings

  digitalWrite(5, HIGH); // Disable 3.3V rail
  digitalWrite(6, LOW);  // Disable 5V rail

  // Disable SPI pins/SD chip select to save power
  pinMode(23, INPUT);
  pinMode(24, INPUT);
  pinMode(10, INPUT);  // Needs to be correct pin for SD CS on Hypnos

  Feather.power_down();
  getSleepManager(Feather).sleep(); // Sketch pauses here until RTC alarm

  digitalWrite(5, LOW); // Enable 3.3V rail
  digitalWrite(6, HIGH);  // Enable 5V rail

  pinMode(23, OUTPUT);
  pinMode(24, OUTPUT);
  pinMode(10, OUTPUT); // Needs to be correct pin for SD CS on Hypnos

  // *** SITS Here in Sleep till p12 RTC Alarm Wake Signal ...

  Feather.power_up();
  delay(1000);        // Delay for power 
}

[udellc]

Edit: We imposed a non-integrated soft fix for creating and posting to the correct topic. The issue fundamentally is NOT resolved and should be addressed in the near future for a true integrated version. Code for soft fix follows:

  // Build JSON document to publish via MQTT
  doc.clear();
  jsonSerialized = "";
  doc.add(Feather.internal_json(false));
  serializeJson(doc, jsonSerialized);
  //publish_mqtt(topic, jsonSerialized);
  publish_mqtt(String(SITE_NAME)+"/Chime"+String(Feather.get_instance_num()), jsonSerialized); // Replace content in Quotes with the Topic name