Make String object size configurable and revise tests to accommodate …

…the largest size. (#1961)

SSO length is stored in 7 bits so can be at most 127 chars.

Sming/README.rst

@@ -81,6 +81,56 @@ The task queue is used for *System.queueCallback()* calls.
    which may not be desirable.
 
 
+String Optimisation
+-------------------
+
+The ``String`` class is probably the most used class in the Arduino world.
+It is also heavily used within Sming.
+
+Unfortunately it gets the blame for one of the most indidious problems in the
+embedded world, `heap fragmentation <https://cpp4arduino.com/2018/11/06/what-is-heap-fragmentation.html>`__.
+
+To alleviate this problem, Sming uses a technique known as *Small String Optimisation*,
+which uses the available space inside the String object itself to avoid using the heap for small allocations
+of 10 characters or fewer.
+
+This was lifted from the `Arduino Esp8266 core <https://github.com/esp8266/arduino/pull/5690>`.
+Superb work - thank you!
+
+We've also added an experimental feature which lets you increase the size of a String object to
+reduce heap allocations further. The effect of this will vary depending on your application,
+but you can see some example figures in :pull-request:`1951`.
+
+Benefits of increasing STRING_OBJECT_SIZE:
+
+-  Increase code speed
+-  Fewer heap allocations
+
+Drawbacks:
+
+-  Increased static memory usage for global/static String objects or embedded within global/static class instances.
+-  A String can use SSO _or_ the heap, but not both together, so when/if it switches to heap mode
+   then any additional space will remain unused, even if the String is itself allocated on the heap.
+
+
+.. envvar:: STRING_OBJECT_SIZE
+
+   minimum: 12 bytes (default)
+   maximum: 128 bytes
+
+   Must be an integer multiple of 4 bytes.
+
+   Allows the size of a String object to be changed to increase the string length available
+   before the heap is used.
+
+   .. note::
+
+      The current implementation uses one byte for a NUL terminator, and another to store the length,
+      so the maximum SSO string length is (STRING_OBJECT_SIZE - 2) characters.
+
+      However, the implementation may change so if you need to check the maximum SSO string size
+      in your code, please use ``String::SSO_CAPACITY``.
+
 
 Release builds
 --------------

Sming/Wiring/WString.cpp

@@ -268,6 +268,7 @@ void String::move(String &rhs)
 	if(!sso.set) {
 		free(ptr.buffer);
 	}
+	sso.set = false;
 	ptr = rhs.ptr;
 	// Can't use rhs.invalidate here as it would free the buffer
 	rhs.ptr.buffer = nullptr;

Sming/Wiring/WString.h

@@ -430,30 +430,34 @@ class String
     long toInt(void) const;
     float toFloat(void) const;
 
+	/// Max chars. (excluding NUL terminator) we can store in SSO mode
+	static constexpr size_t SSO_CAPACITY = STRING_OBJECT_SIZE - 2;
+
 protected:
     /// Used when contents allocated on heap
 	struct PtrBuf {
 		char* buffer;		  // the actual char array
 		size_t len;			  // the String length (not counting the '\0')
 		size_t capacity : 31; // the array length minus one (for the '\0')
-		size_t isSSO : 1;
 	};
 	// For small strings we can store data directly without requiring the heap
-	static constexpr size_t SSO_CAPACITY = sizeof(PtrBuf) - 2; ///< Less one char for '\0'
 	struct SsoBuf {
 		char buffer[SSO_CAPACITY + 1];
 		unsigned char len : 7;
 		unsigned char set : 1; ///< true for SSO mode
 	};
 	union {
 		struct {
-			size_t u32[3] = {0};
+			size_t u32[STRING_OBJECT_SIZE / 4] = {0};
 		};
 		PtrBuf ptr;
 		SsoBuf sso;
 	};
 
-	static_assert(sizeof(PtrBuf) == sizeof(SsoBuf), "String size incorrect - check alignment");
+	static_assert(STRING_OBJECT_SIZE == sizeof(SsoBuf), "SSO Buffer alignment problem");
+	static_assert(STRING_OBJECT_SIZE >= sizeof(PtrBuf), "STRING_OBJECT_SIZE too small");
+	static_assert(STRING_OBJECT_SIZE <= 128, "STRING_OBJECT_SIZE too large (max. 128)");
+	static_assert(STRING_OBJECT_SIZE % 4 == 0, "STRING_OBJECT_SIZE must be a multiple of 4");
 
 protected:
 	// Free any heap memory and set to non-SSO mode; isNull() will return true

Sming/component.mk

@@ -140,3 +140,8 @@ endif
 COMPONENT_VARS		+= TASK_QUEUE_LENGTH
 TASK_QUEUE_LENGTH	?= 10
 COMPONENT_CXXFLAGS	+= -DTASK_QUEUE_LENGTH=$(TASK_QUEUE_LENGTH)
+
+# Size of a String object - change this to increase space for Small String Optimisation (SSO)
+COMPONENT_VARS		+= STRING_OBJECT_SIZE
+STRING_OBJECT_SIZE	?= 12
+GLOBAL_CFLAGS		+= -DSTRING_OBJECT_SIZE=$(STRING_OBJECT_SIZE) 

tests/HostTests/app/arduino-test-string.cpp

@@ -285,103 +285,23 @@ class ArduinoStringTest : public TestGroup
 
 		TEST_CASE("String SSO works", "[core][String]")
 		{
-			// This test assumes that SSO_SIZE==8, if that changes the test must as well
-			String s;
-			s += "0";
-			REQUIRE(s == "0");
-			REQUIRE(s.length() == 1);
+			PSTR_ARRAY(whole, "0123456789abcdefghijklmnopqrstuvwxyz"
+							  "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+							  "0123456789abcdefghijklmnopqrstuvwxyz"
+							  "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ");
+
+			// Keep a pointer to the static buffer which must not change
+			String s = "";
 			const char* savesso = s.c_str();
-			s += 1;
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "01");
-			REQUIRE(s.length() == 2);
-			s += "2";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "012");
-			REQUIRE(s.length() == 3);
-			s += 3;
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "0123");
-			REQUIRE(s.length() == 4);
-			s += "4";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "01234");
-			REQUIRE(s.length() == 5);
-			s += "5";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "012345");
-			REQUIRE(s.length() == 6);
-			s += "6";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "0123456");
-			REQUIRE(s.length() == 7);
-			s += "7";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "01234567");
-			REQUIRE(s.length() == 8);
-			s += "8";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "012345678");
-			REQUIRE(s.length() == 9);
-			s += "9";
-			REQUIRE(s.c_str() == savesso);
-			REQUIRE(s == "0123456789");
-			REQUIRE(s.length() == 10);
-			if(sizeof(savesso) == 4) {
-				s += "a";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789a");
-				REQUIRE(s.length() == 11);
-				s += "b";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789ab");
-				REQUIRE(s.length() == 12);
-				s += "c";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abc");
-				REQUIRE(s.length() == 13);
-			} else {
-				s += "a";
-				REQUIRE(s.c_str() == savesso);
-				REQUIRE(s == "0123456789a");
-				REQUIRE(s.length() == 11);
-				s += "bcde";
-				REQUIRE(s.c_str() == savesso);
-				REQUIRE(s == "0123456789abcde");
-				REQUIRE(s.length() == 15);
-				s += "fghi";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghi");
-				REQUIRE(s.length() == 19);
-				s += "j";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghij");
-				REQUIRE(s.length() == 20);
-				s += "k";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghijk");
-				REQUIRE(s.length() == 21);
-				s += "l";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghijkl");
-				REQUIRE(s.length() == 22);
-				s += "m";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghijklm");
-				REQUIRE(s.length() == 23);
-				s += "nopq";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghijklmnopq");
-				REQUIRE(s.length() == 27);
-				s += "rstu";
-				REQUIRE(s.c_str() != savesso);
-				REQUIRE(s == "0123456789abcdefghijklmnopqrstu");
-				REQUIRE(s.length() == 31);
+			for(unsigned i = 0; i < String::SSO_CAPACITY + 5; ++i) {
+				s += whole[i];
+				REQUIRE(s.equals(whole, i + 1));
+				if(i < String::SSO_CAPACITY) {
+					REQUIRE(s.c_str() == savesso);
+				} else {
+					REQUIRE(s.c_str() != savesso);
+				}
 			}
-			s = "0123456789abcde";
-			s = s.substring(s.indexOf('a'));
-			REQUIRE(s == "abcde");
-			REQUIRE(s.length() == 5);
 		}
 
 		auto repl = [](const String& key, const String& val, String& s, boolean useURLencode) { s.replace(key, val); };
@@ -501,17 +421,17 @@ class ArduinoStringTest : public TestGroup
 		{
 			String s, l;
 			// Make these large enough to span SSO and non SSO
-			String whole = "#123456789012345678901234567890";
-			const char* res = "abcde123456789012345678901234567890";
+#define C10 "1234567890"
+#define C60 C10 C10 C10 C10 C10 C10
+#define C140 C60 C60 C10 C10
+			String whole = F("#" C140);
+			String res = F("abcde" C140);
 			for(size_t i = 1; i < whole.length(); i++) {
 				s = whole.substring(0, i);
 				l = s;
 				l.replace("#", "abcde");
-				char buff[64];
-				strcpy(buff, res);
-				buff[5 + i - 1] = 0;
-				REQUIRE(!strcmp(l.c_str(), buff));
-				REQUIRE(l.length() == strlen(buff));
+				auto reslen = 5 + i - 1;
+				REQUIRE(l.equals(res.c_str(), reslen));
 			}
 		}
 	}