Add Basic_Templates sample with CSV reader class (#2403)

This PR aims to provide a clearer example of how to use templates.

It also updates the Formatter classes:

- JSON string escaping needs to escape double-quotes, \"
- Standard format `quote` method needs to first escape any embedded quotes. Uses "" rather than \" as it's CSV-compatible.
- Add XML formatter with basic escaping

[scan:coverity]

Sming/Arch/Esp32/standard.hw

@@ -27,7 +27,7 @@
 		},
 		"factory": {
 			"address": "0x010000",
-			"size": "0x180000",
+			"size": "0x0f0000",
 			"type": "app",
 			"subtype": "factory",
 			"filename": "$(TARGET_BIN)"

Sming/Arch/Rp2040/Components/spi_flash/flashmem.cpp

@@ -19,6 +19,9 @@
 #define FLASHCMD_READ_SFDP 0x5a
 #define FLASHCMD_READ_JEDEC_ID 0x9f
 
+// Buffers need to be word aligned for flash access
+#define ATTR_ALIGNED __attribute__((aligned(4)))
+
 namespace
 {
 /*
@@ -233,14 +236,14 @@ uint32_t flashmem_write(const void* from, uint32_t toaddr, uint32_t size)
 
 uint32_t flashmem_read(void* to, uint32_t fromaddr, uint32_t size)
 {
-	if(IS_ALIGNED(fromaddr) && IS_ALIGNED(size)) {
+	if(IS_ALIGNED(to) && IS_ALIGNED(fromaddr) && IS_ALIGNED(size)) {
 		return readAligned(to, fromaddr, size);
 	}
 
 	const uint32_t blksize = flashReadUnitSize;
 	const uint32_t blkmask = flashReadUnitSize - 1;
 
-	uint8_t tmpdata[flashBufferCount * blksize];
+	ATTR_ALIGNED uint8_t tmpdata[flashBufferCount * blksize];
 	auto pto = static_cast<uint8_t*>(to);
 	size_t remain = size;
 

Sming/Core/Data/CsvReader.cpp

@@ -0,0 +1,123 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * CsvReader.cpp
+ *
+ * @author: 2021 - Mikee47 <[email protected]>
+ *
+ ****/
+
+#include "CsvReader.h"
+
+void CsvReader::reset()
+{
+	source->seekFrom(0, SeekOrigin::Start);
+	if(!userHeadingsProvided) {
+		readRow();
+		headings = row;
+	}
+	row = nullptr;
+}
+
+bool CsvReader::readRow()
+{
+	constexpr size_t blockSize{512};
+
+	String buffer(std::move(reinterpret_cast<String&>(row)));
+	constexpr char quoteChar{'"'};
+	enum class FieldKind {
+		unknown,
+		quoted,
+		unquoted,
+	};
+	FieldKind fieldKind{};
+	bool escape{false};
+	bool quote{false};
+	char lc{'\0'};
+	unsigned writepos{0};
+
+	while(true) {
+		if(buffer.length() == maxLineLength) {
+			debug_w("[CSV] Line buffer limit reached %u", maxLineLength);
+			return false;
+		}
+		size_t buflen = std::min(writepos + blockSize, maxLineLength);
+		if(!buffer.setLength(buflen)) {
+			debug_e("[CSV] Out of memory %u", buflen);
+			return false;
+		}
+		auto len = source->readBytes(buffer.begin() + writepos, buflen - writepos);
+		if(len == 0) {
+			if(writepos == 0) {
+				return false;
+			}
+			buffer.setLength(writepos);
+			row = std::move(buffer);
+			return true;
+		}
+		buflen = writepos + len;
+		unsigned readpos = writepos;
+
+		for(; readpos < buflen; ++readpos) {
+			char c = buffer[readpos];
+			if(escape) {
+				switch(c) {
+				case 'n':
+					c = '\n';
+					break;
+				case 'r':
+					c = '\r';
+					break;
+				case 't':
+					c = '\t';
+					break;
+				default:;
+					// Just accept character
+				}
+				escape = false;
+			} else {
+				if(fieldKind == FieldKind::unknown) {
+					if(c == quoteChar) {
+						fieldKind = FieldKind::quoted;
+						quote = true;
+						lc = '\0';
+						continue;
+					}
+					fieldKind = FieldKind::unquoted;
+				}
+				if(c == quoteChar) {
+					quote = !quote;
+					if(fieldKind == FieldKind::quoted) {
+						if(lc == quoteChar) {
+							buffer[writepos++] = c;
+							lc = '\0';
+						} else {
+							lc = c;
+						}
+						continue;
+					}
+				} else if(c == '\\') {
+					escape = true;
+					continue;
+				} else if(!quote) {
+					if(c == fieldSeparator) {
+						c = '\0';
+						fieldKind = FieldKind::unknown;
+					} else if(c == '\r') {
+						continue;
+					} else if(c == '\n') {
+						source->seekFrom(readpos + 1 - buflen, SeekOrigin::Current);
+						buffer.setLength(writepos);
+						row = std::move(buffer);
+						return true;
+					}
+				}
+			}
+			buffer[writepos++] = c;
+			lc = c;
+		}
+	}
+}

Sming/Core/Data/CsvReader.h

@@ -0,0 +1,144 @@
+/****
+ * Sming Framework Project - Open Source framework for high efficiency native ESP8266 development.
+ * Created 2015 by Skurydin Alexey
+ * http://github.com/SmingHub/Sming
+ * All files of the Sming Core are provided under the LGPL v3 license.
+ *
+ * CsvReader.h
+ *
+ * @author: 2021 - Mikee47 <[email protected]>
+ *
+ ****/
+
+#pragma once
+
+#include "Stream/DataSourceStream.h"
+#include "CStringArray.h"
+#include <memory>
+
+/**
+ * @brief Class to parse a CSV file
+ *
+ * Spec: https://www.ietf.org/rfc/rfc4180.txt
+ *
+ * 1. Each record is located on a separate line
+ * 2. Line ending for last record in the file is optional
+ * 3. Field headings are provided either in the source data or in constructor (but not both)
+ * 4. Fields separated with ',' and whitespace considered part of field content
+ * 5. Fields may or may not be quoted - if present, will be removed during parsing
+ * 6. Fields may contain line breaks, quotes or commas
+ * 7. Quotes may be escaped thus "" if field itself is quoted
+ *
+ * Additional features:
+ *
+ * - Line breaks can be \n or \r\n
+ * - Escapes codes within fields will be converted: \n \r \t \", \\
+ * - Field separator can be changed in constructor
+ */
+class CsvReader
+{
+public:
+	/**
+	 * @brief Construct a CSV reader
+	 * @param source Stream to read CSV text from
+	 * @param fieldSeparator
+	 * @param headings Required if source data does not contain field headings as first row
+	 * @param maxLineLength Limit size of buffer to guard against malformed data
+	 */
+	CsvReader(IDataSourceStream* source, char fieldSeparator = ',', const CStringArray& headings = nullptr,
+			  size_t maxLineLength = 2048)
+		: fieldSeparator(fieldSeparator), userHeadingsProvided(headings), maxLineLength(maxLineLength),
+		  headings(headings)
+	{
+		this->source.reset(source);
+		reset();
+	}
+
+	/**
+	 * @brief Reset reader to start of CSV file
+	 *
+	 * Cursor is set to 'before start'.
+	 * Call 'next()' to fetch first record.
+	 */
+	void reset();
+
+	/**
+	 * @brief Seek to next record
+	 */
+	bool next()
+	{
+		return readRow();
+	}
+
+	/**
+	 * @brief Get number of columns
+	 */
+	unsigned count() const
+	{
+		return headings.count();
+	}
+
+	/**
+	 * @brief Get a value from the current row
+	 * @param index Column index, starts at 0
+	 * @retval const char* nullptr if index is not valid
+	 */
+	const char* getValue(unsigned index)
+	{
+		return row[index];
+	}
+
+	/**
+	 * @brief Get a value from the current row
+	 * @param index Column name
+	 * @retval const char* nullptr if name is not found
+	 */
+	const char* getValue(const char* name)
+	{
+		return getValue(getColumn(name));
+	}
+
+	/**
+	 * @brief Get index of columnn given its name
+	 * @param name Column name to find
+	 * @retval int -1 if name is not found
+	 */
+	int getColumn(const char* name)
+	{
+		return headings.indexOf(name);
+	}
+
+	/**
+	 * @brief Determine if row is valid
+	 */
+	explicit operator bool() const
+	{
+		return bool(row);
+	}
+
+	/**
+	 * @brief Get headings
+	 */
+	const CStringArray& getHeadings() const
+	{
+		return headings;
+	}
+
+	/**
+	 * @brief Get current row
+	 */
+	const CStringArray& getRow() const
+	{
+		return row;
+	}
+
+private:
+	bool readRow();
+
+	std::unique_ptr<IDataSourceStream> source;
+	char fieldSeparator;
+	bool userHeadingsProvided;
+	size_t maxLineLength;
+	CStringArray headings;
+	CStringArray row;
+};

Sming/Core/Data/Format.h

@@ -17,5 +17,6 @@
 #include "Format/Standard.h"
 #include "Format/Html.h"
 #include "Format/Json.h"
+#include "Format/Xml.h"
 
 using Formatter = Format::Formatter;

Sming/Core/Data/Format/Json.cpp

@@ -76,6 +76,8 @@ void Json::escape(String& value) const
 				c = '_';
 		}
 	}
+
+	value.replace("\"", "\\\"");
 }
 
 } // namespace Format

Sming/Core/Data/Format/Standard.cpp

@@ -15,4 +15,28 @@
 namespace Format
 {
 Standard standard;
+
+void Standard::quote(String& value) const
+{
+	value.replace("\"", "\"\"");
+	char cQuote{'"'};
+	auto len = value.length();
+	value.setLength(len + 2);
+	memmove(&value[1], value.c_str(), len);
+	value[0] = cQuote;
+	value[len + 1] = cQuote;
+}
+
+void Standard::unQuote(String& value) const
+{
+	char quote = value[0];
+	if(quote == '"' || quote == '\'') {
+		auto len = value.length();
+		if(len > 1 && value[len - 1] == quote) {
+			value.remove(len - 1, 1);
+			value.remove(0, 1);
+		}
+	}
 }
+
+} // namespace Format

Sming/Core/Data/Format/Standard.h

@@ -23,27 +23,8 @@ class Standard : public Formatter
 	{
 	}
 
-	void quote(String& value) const override
-	{
-		char cQuote{'"'};
-		auto len = value.length();
-		value.setLength(len + 2);
-		memmove(&value[1], value.c_str(), len);
-		value[0] = cQuote;
-		value[len + 1] = cQuote;
-	}
-
-	void unQuote(String& value) const override
-	{
-		char quote = value[0];
-		if(quote == '"' || quote == '\'') {
-			auto len = value.length();
-			if(len > 1 && value[len - 1] == quote) {
-				value.remove(len - 1, 1);
-				value.remove(0, 1);
-			}
-		}
-	}
+	void quote(String& value) const override;
+	void unQuote(String& value) const override;
 
 	MimeType mimeType() const override
 	{