UPnP code generation (#2154)

Further development to UPnP framework.
Some additions required to the build system to support intermediate code generation.

* Add COMPONENT_PREREQUISITES and COMPONENT_BUILD_BASE to build framework
* Fix `ListAllSubDirs` - should be able to handle multiple arguments
* Add `ListAllFiles` and `dirx`
* Add UPnP-Schema library
* Update SSDP, UPnP, HueEmulator libraries

.gitmodules

@@ -271,6 +271,10 @@
 	path = Sming/Libraries/UPnP
 	url = https://github.com/mikee47/Sming-UPnP
 	ignore = dirty
+[submodule "Sming/Libraries/UPnP-Schema"]
+	path = Sming/Libraries/UPnP-Schema
+	url = https://github.com/mikee47/UPnP-Schema
+	ignore = dirty
 [submodule "Libraries.VT100"]
 	path = Sming/Libraries/VT100
 	url = https://github.com/mikee47/VT100

Sming/Core/Data/Stream/MemoryDataStream.cpp

@@ -21,6 +21,10 @@ MemoryDataStream::MemoryDataStream(String&& string) noexcept
 bool MemoryDataStream::ensureCapacity(size_t minCapacity)
 {
 	if(capacity < minCapacity) {
+		if(minCapacity > maxCapacity) {
+			debug_e("MemoryDataStream too large, requested %u limit is %u", minCapacity, maxCapacity);
+			return false;
+		}
 		size_t newCapacity = minCapacity;
 		if(capacity != 0) {
 			// If expanding stream, increase buffer capacity in anticipation of further writes
@@ -30,6 +34,7 @@ bool MemoryDataStream::ensureCapacity(size_t minCapacity)
 		// realloc can fail, store the result in temporary pointer
 		auto newBuffer = (char*)realloc(buffer, newCapacity);
 		if(newBuffer == nullptr) {
+			debug_e("MemoryDataStream realloc(%u) failed", newCapacity);
 			return false;
 		}
 

Sming/Core/Data/Stream/MemoryDataStream.h

@@ -26,7 +26,9 @@
 class MemoryDataStream : public ReadWriteStream
 {
 public:
-	MemoryDataStream() = default;
+	MemoryDataStream(size_t maxCapacity = UINT16_MAX) : maxCapacity(maxCapacity)
+	{
+	}
 
 	/**
 	 * @brief Stream takes ownership of String content using move semantics
@@ -103,8 +105,9 @@ class MemoryDataStream : public ReadWriteStream
 	}
 
 private:
-	char* buffer = nullptr; ///< Stream content stored here
-	size_t readPos = 0;		///< Offset to current read position
-	size_t size = 0;		///< Number of bytes stored in stream (i.e. the write position)
-	size_t capacity = 0;	///< Number of bytes allocated in buffer
+	char* buffer = nullptr;			///< Stream content stored here
+	size_t maxCapacity{UINT16_MAX}; ///< Limit size of stream
+	size_t readPos = 0;				///< Offset to current read position
+	size_t size = 0;				///< Number of bytes stored in stream (i.e. the write position)
+	size_t capacity = 0;			///< Number of bytes allocated in buffer
 };

Sming/Core/Data/Stream/ReadWriteStream.h

@@ -24,6 +24,8 @@ class ReadWriteStream : public IDataSourceStream
 		return write(&charToWrite, 1);
 	}
 
+	using Print::write;
+
 	/** @brief  Write chars to stream
      *  @param  buffer Pointer to buffer to write to the stream
      *  @param  size Quantity of chars to write

Sming/Wiring/FIFO.h

@@ -73,8 +73,10 @@ template <typename T, int rawSize> bool FIFO<T, rawSize>::enqueue(T element)
 	}
 	numberOfElements++;
 	raw[nextIn] = element;
-	if(++nextIn >= rawSize) // advance to next index, wrap if needed
+	// advance to next index, wrap if needed
+	if(++nextIn >= rawSize) {
 		nextIn = 0;
+	}
 	return true;
 }
 

Sming/build.mk

@@ -249,14 +249,30 @@ define IsSubDir
 $(if $(subst $(1:/=),,$(2:/=)),$(findstring $(1:/=),$2),)
 endef
 
+# List sub-directories recursively for a single root directory
+# Results are sorted and without trailing path separator
+# Sub-directories with spaces are skipped
+# $1 -> Root path
+define ListAllSubDirsSingle
+$(foreach d,$(dir $(wildcard $1/*/.)),$(if $(call IsSubDir,$1,$d),$(d:/=) $(call ListAllSubDirs,$(d:/=))))
+endef
+
 # List sub-directories recursively for a list of root directories
 # Results are sorted and without trailing path separator
 # Sub-directories with spaces are skipped
 # $1 -> Root paths
 define ListAllSubDirs
-$(foreach d,$(dir $(wildcard $1/*/.)),$(if $(call IsSubDir,$1,$d),$(d:/=) $(call ListAllSubDirs,$(d:/=))))
+$(foreach d,$1,$(call ListAllSubDirsSingle,$d))
 endef
 
+# Recursively search list of directories for matching files
+# $1 -> Directories to scan
+# $2 -> Filename filter
+define ListAllFiles
+$(wildcard $(foreach d,$(call ListAllSubDirs,$1),$d/$2))
+endef
+
+
 # Display variable and list values, e.g. $(call PrintVariable,LIBS)
 # $1 -> Name of variable containing values
 define PrintVariable
@@ -276,6 +292,11 @@ define PrintVariableRefs
 	$(foreach item,$(sort $($1)),$(info - $(item) = $(value $(item))) )
 endef
 
+#
+# Get directory without trailing separator
+# $1 -> List of directories
+dirx = $(patsubst %/,%,$(dir $1))
+
 # Extract commented target information from makefiles and display
 # Based on code from https://suva.sh/posts/well-documented-makefiles/
 define PrintHelp

Sming/building.rst

@@ -468,6 +468,14 @@ These values are for reference only and should not be modified.
    This should be used if the Component provides any application code or targets to ensure it is
    built in the correct directory (but not by this makefile).
 
+   This value changes depending on the build variant.
+
+.. envvar:: COMPONENT_BUILD_BASE
+
+   This value does not change with build variant.
+
+   If the Component generates source code, for example, it can be placed here (in a sub-directory).
+
 .. envvar:: COMPONENT_LIBDIR
 
    Location to store created Component (shared) libraries
@@ -500,6 +508,11 @@ changed as required.
    If targets should be built for each application, use :envvar:`CUSTOM_TARGETS` instead.
    See :component:`spiffs` for an example.
 
+.. envvar:: COMPONENT_PREREQUISITES
+
+   These targets will be built before anything else. If your library generates source code,
+   for example, then it should be done by setting this value to the appropriate targets.
+
 .. envvar:: COMPONENT_RULE
 
    This is a special value used to prefix any custom targets which are to be built as

Sming/project.mk

@@ -160,9 +160,11 @@ COMPONENT_INCDIRS		:= include
 COMPONENT_NAME			:= $1
 COMPONENT_LIBNAME		:= $1
 CMP_$1_BUILD_BASE		:= $3/$1
+COMPONENT_BUILD_BASE	:= $$(CMP_$1_BUILD_BASE)
 COMPONENT_BUILD_DIR		:= $$(CMP_$1_BUILD_BASE)
 COMPONENT_VARS			:=
 COMPONENT_RELINK_VARS	:=
+COMPONENT_PREREQUISITES	:=
 COMPONENT_TARGETS		:=
 COMPONENT_DEPENDS		:=
 COMPONENT_PYTHON_REQUIREMENTS		:= $$(wildcard $2/requirements.txt)
@@ -184,6 +186,7 @@ LIBS					+= $$(EXTRA_LIBS)
 CMP_$1_LDFLAGS			:= $$(EXTRA_LDFLAGS)
 LDFLAGS					+= $$(CMP_$1_LDFLAGS)
 endif
+CMP_$1_PREREQUISITES	:= $$(COMPONENT_PREREQUISITES)
 CMP_$1_TARGETS			:= $$(COMPONENT_TARGETS)
 CMP_$1_BUILD_DIR		:= $$(COMPONENT_BUILD_DIR)
 CMP_$1_LIBNAME			:= $$(COMPONENT_LIBNAME)
@@ -266,7 +269,7 @@ $(foreach c,$(COMPONENTS),$(eval $(call ResolveDependencies,$c)))
 # $1 => Component name
 define CheckComponentMatches
 ifneq ($1,Sming)
-COMPONENT_MATCHES := $(filter %/$1,$(ALL_COMPONENT_DIRS))
+COMPONENT_MATCHES := $(sort $(filter %/$1,$(ALL_COMPONENT_DIRS)))
 ifeq ($$(words $$(COMPONENT_MATCHES)),0)
 $$(warning No matches found for Component '$1')
 else ifneq ($$(words $$(COMPONENT_MATCHES)),1)
@@ -362,6 +365,7 @@ $1-build: $(addsuffix -build,$(filter $(CMP_$1_DEPENDS),$(BUILDABLE_COMPONENTS))
 	+$(Q) $(MAKE) -r -R --no-print-directory -C $(CMP_$1_BUILD_DIR) -f $(SMING_HOME)/component-wrapper.mk \
 		COMPONENT_NAME=$1 \
 		COMPONENT_PATH=$(CMP_$1_PATH) \
+		COMPONENT_BUILD_BASE=$(CMP_$1_BUILD_BASE) \
 		COMPONENT_LIBDIR=$(CMP_$1_LIBDIR) \
 		COMPONENT_LIBNAME=$(CMP_$1_LIBNAME) \
 		COMPONENT_LIBHASH=$(CMP_$1_LIBHASH) \
@@ -432,9 +436,12 @@ checkdirs: | $(BUILD_BASE) $(FW_BASE) $(TOOLS_BASE) $(APP_LIBDIR) $(USER_LIBDIR)
 $(BUILD_BASE) $(FW_BASE) $(TOOLS_BASE) $(APP_LIBDIR) $(USER_LIBDIR):
 	$(Q) mkdir -p $@
 
+# Targets to be built before anything else (e.g. source code generators)
+PREREQUISITES := $(foreach c,$(COMPONENTS),$(CMP_$c_PREREQUISITES))
+
 # Build all Component (user) libraries
 .PHONY: components
-components: $(ALL_COMPONENT_TARGETS) $(CUSTOM_TARGETS)
+components: $(PREREQUISITES) $(ALL_COMPONENT_TARGETS) $(CUSTOM_TARGETS)
 
 ##@Cleaning
 
@@ -523,6 +530,7 @@ list-config: ##Print the contents of build variables
 	$(info ** Sming build configuration **)
 	$(info )
 	$(if $(V),$(call PrintVariable,MAKEFILE_LIST))
+	$(call PrintVariableSorted,PREREQUISITES)
 	$(call PrintVariableSorted,CUSTOM_TARGETS)
 	$(call PrintVariableSorted,LIBS)
 	$(call PrintVariableSorted,ARDUINO_LIBRARIES)