WeatherChimes Project SD Hard Faults

[udellc]

Describe the bug
WeatherChimes project; System runs as expected for hours, to many days, then experiences periodic Hard Faults.

Example of Errors file output:
name | instance | Problem. | Corrupted | Failures | File_Source | Line_Number
Chime | 4. | Hard_Fault | 0 | 1 | SD.cpp | 360

Data continues to log to CSV file after Hard Faults
While SD data looks good, the system should not experience known hard faults before release to field operation.

Hardware in Use
Chime #4 (Chime #3 seems to be experiencing additional SDI-12 related hardware issues)
Feather M0 WiFi
Hypnos Board: SD, DS3231 RTC, 3v 5v Power rail switching
TSL2591
GS3 (SDI-12 soil moisture sensor by Meter)
SHT-30 Air Temp, Humidity

To Reproduce
Steps to reproduce the behavior:
Change WiFi router info in arduino_secrets for: SECRET_SSID, SECRET_PASS for your internet router
Ask Will or Chet for arduino secrets for MQTT broker and input
Upload attached WeatherChimesV4SDSleepMQTT.ino and config.h to WeatherChimes Hardware

Expected behavior
System should measure sensors, log to SD, and publish data to MongoDB via MQTT with no hard faults or hanging.

Code

///////////////////////////////////////////////////////////////////////////////

// This is a basic example that demonstrates how to log data to an SD card
// using Loom.

///////////////////////////////////////////////////////////////////////////////

#include <Loom.h>
#include "MQTT.h"

// These are default times to sleep between cycles if there is no SD_config.txt file exists on SD card for user-set time
int secs = 20;
int mins = 0;
int hours = 0;
int days = 0;

// Include configuration
const char* json_config =
#include "config.h"
;

// In Tools menu, set:
// Internet  > Disabled
// Sensors   > Enabled
// Radios    > Disabled
// Actuators > Disabled
// Max       > Disabled

using namespace Loom;

Loom::Manager Feather{};

char devName[20];
String topic = "";
String jsonSerialized = "";

DynamicJsonDocument doc(1024);

void setup()
{
  delay(3000); // Wait a bit in case we want to program the device
  pinMode(5, OUTPUT);   // Enable control of 3.3V rail 
  pinMode(6, OUTPUT);   // Enable control of 5V rail 

  //See Above
  digitalWrite(5, LOW); // Enable 3.3V rail
  digitalWrite(6, HIGH);  // Enable 5V rail
  
  Feather.begin_LED();
  Feather.begin_serial(true);
  Feather.parse_config(json_config);
  Feather.print_config();

  // Register an interrupt on the RTC alarm pin
  getInterruptManager(Feather).register_ISR(12, wakeISR_RTC, LOW, ISR_Type::IMMEDIATE);

  // Get the device name and then using the device name, instance number and site name create a topic name to publish to
  Feather.get_device_name(devName);
  topic = String(SITE_NAME) + "/" + String(devName) + String(Feather.get_instance_num());

  enable_wifi();

  LPrintln("\n ** Setup Complete ** ");
}


void loop()
{
  Feather.measure();
  Feather.package();
  Feather.display_data();

  // Log using default filename as provided in configuration
  // in this case, 'datafile.csv'
  getSD(Feather).log();

  // Or log to a specific file (does not change what default file is set to)
  // getSD(Feather)log("specific.csv");

  // Log online to MongoDB via MQTT WiFi
  connect_to_wifi();
  connect_to_broker(1);
  // Build JSON document to publish via MQTT
  doc.clear();
  jsonSerialized = "";
  doc.add(Feather.internal_json(false));
  serializeJson(doc, jsonSerialized);
  publish_mqtt(topic, jsonSerialized);

  // Set RTC Timer for wake up and sample to next time interval
  getInterruptManager(Feather).RTC_alarm_duration(TimeSpan(days,hours,mins,secs));
  getInterruptManager(Feather).reconnect_interrupt(12);

  disconnect_wifi(); // Disable WiFi for power savings

  digitalWrite(5, HIGH); // Disable 3.3V rail
  digitalWrite(6, LOW);  // Disable 5V rail

  // Disable SPI pins/SD chip select to save power
  pinMode(23, INPUT);
  pinMode(24, INPUT);
  pinMode(10, INPUT);  // Needs to be correct pin for SD CS on Hypnos

  Feather.power_down();
  getSleepManager(Feather).sleep(); // Sketch pauses here until RTC alarm

  digitalWrite(5, LOW); // Enable 3.3V rail
  digitalWrite(6, HIGH);  // Enable 5V rail

  pinMode(23, OUTPUT);
  pinMode(24, OUTPUT);
  pinMode(10, OUTPUT); // Needs to be correct pin for SD CS on Hypnos

  // *** SITS Here in Sleep till p12 RTC Alarm Wake Signal ...

  Feather.power_up();
  delay(1000);        // Delay for power 
}

void wakeISR_RTC() {  
  // disable the interrupt
  detachInterrupt(12);
}

Config

"{\
  'general':\
  {\
    'name':'Chime',\
    'instance':3,\
    'interval':5000,\
    'print_verbosity':2\
  },\
  'components':[\
    {\
      'name':'SD',\
      'params':[true, 1000, 10,'chimes', true]\
    },\
    {\
      'name':'InterruptManager',\
      'params':'default'\
    },\
    {\
      'name':'SleepManager',\
      'params':[true,false,1]\
    },\
    {\
      'name':'DS3231',\
      'params':[10, true, true]\
    },\
    {\
      'name':'SDIManager',\
      'params':'default'\
    },\
    {\
      'name':'TSL2591',\
      'params':'default'\
    },\
    {\
      'name':'SHT31D',\
      'params':'default'\
    },\
    {\
      'name':'Analog',\
      'params':[\
        8,\
        12,\
        false,\
        false,\
        false,\
        false,\
        false,\
        false,\
        1,\
        1,\
        1,\
        1,\
        1,\
        1,\
        25.0\
      ]\
    }\
  ]\
}"

MQTT.h

#pragma once
#include <ArduinoMqttClient.h>
#include <WiFi101.h>

#include "arduino_secrets.h"

// Feather M0 Wifi Pins
#define WINC_CS   8
#define WINC_IRQ  7
#define WINC_RST  4
#define WINC_EN   2

char ssid[] = SECRET_SSID;
char pass[] = SECRET_PASS;

const char broker[] = SECRET_BROKER;
int port = BROKER_PORT;

WiFiClient wifiClient;
MqttClient mqttClient(wifiClient);

/* Disconnect from broker and WiFi*/
void disconnect_wifi(){
  mqttClient.stop();
  WiFi.disconnect();
  WiFi.end();
}

/**
 * Connect to the wifi network
 */ 
void connect_to_wifi(){

    // Try to conect to the wifi network
    Serial.print("[MQTT] Attempting to connect to SSID: ");
    Serial.println(ssid);

    // Check if there is a password for the wifi or not
    if(strlen(pass) > 0){
        // Try to connect to the Wifi network until it succeeds 
        while (WiFi.begin(ssid, pass) != WL_CONNECTED) {
            Serial.print("[MQTT] Attempting connection to AP...");
            delay(5000);
        }
    }
    else{
        // Try to connect to the Wifi network until it succeeds 
        while (WiFi.begin(ssid) != WL_CONNECTED) {
            Serial.print("[MQTT] Attempting connection to AP...");
            delay(5000);
        }
    }

    Serial.println("Connected to network!");
}

/*
 * Enable the Wifi but don't connect to any network
 */
void enable_wifi(){
  
  // Set the pins that the WiFi module should
  WiFi.setPins(WINC_CS, WINC_IRQ, WINC_RST, WINC_EN);
  
  // Initialise the Client
  Serial.print(F("\nInit the WiFi module..."));
  
  // Check for the presence of the breakout
  if (WiFi.status() == WL_NO_SHIELD) {
    Serial.println("WINC1500 not present");
    
    // don't continue:
    while (true);
  }

  // Set the WiFi chip into the lowest power mode to conserve energy
  WiFi.maxLowPowerMode();
  
  Serial.println("ATWINC OK!");
}

/**
 * Connect to the MQTT broker
 */ 
void connect_to_broker(int keepAliveMins){

    // Set the MQTT username and password
    mqttClient.setUsernamePassword(BROKER_USER, BROKER_PASSWORD);

    // Set keep alive time
    mqttClient.setKeepAliveInterval(1000 * 60 * keepAliveMins);

    Serial.print("[MQTT] Attempting to connect to broker: ");
    Serial.println(broker);

    // Try to connect to the broker
    if (!mqttClient.connect(broker, port)) {
        Serial.print("[MQTT] MQTT connection failed! Error code = ");
        Serial.println(mqttClient.connectError());
    }

    Serial.println("[MQTT] You're connected to the MQTT broker!");
}

/**
 * Publish the data to the correct topic
 */ 
void publish_mqtt(String topic, String data){

    // send message, the Print interface can be used to set the message contents
    mqttClient.poll();
    delay(250);         // ****** Added 6.8.22 Test if delay after poll fixes freezing
    mqttClient.beginMessage(topic);
    mqttClient.print(data);
    mqttClient.endMessage();
}

arduino_secrets.h

// Wifi settings
#define SECRET_SSID "routername" // WiFi Name
#define SECRET_PASS ""  // WiFi Password

// MQTT Settings
#define BROKER_USER ""
#define BROKER_PASSWORD ""
#define SECRET_BROKER ""
#define BROKER_PORT 1883
#define SITE_NAME "WeatherChimes" // The name of the location where these nodes will be placed

Additional context
Add any other context about the problem here.

Errors.csv
chimes170.csv

[udellc]

Fixed, Hypnos on Chime 4 is different version and CS pin assignment.

[udellc]

Nevermind,
WeatherChimes continues to hard fault as described in the original post. The system recovers with the reset and will operate a while before faulting again.
Logs to SD and MQTT are successful.
But on SD, data is distributed to a lot of files because a new file is created each time the system resets. New files are not a big deal, but we need to figure out the hardfault issue.

[udellc]

Errors.csv