From 544918242de8c0552df6907dc11940cdf3269e65 Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Sun, 8 Jan 2023 15:56:19 +0100 Subject: [PATCH 01/10] Supporting ETag http headers on static files --- libraries/WebServer/src/WebServer.cpp | 9 ++++- libraries/WebServer/src/WebServer.h | 6 +++ .../src/detail/RequestHandlersImpl.h | 40 +++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index 66c01198fc9..fc1cad571fa 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -37,6 +37,7 @@ static const char qop_auth[] PROGMEM = "qop=auth"; static const char qop_auth_quoted[] PROGMEM = "qop=\"auth\""; static const char WWW_Authenticate[] = "WWW-Authenticate"; static const char Content_Length[] = "Content-Length"; +static const char ETAG_HEADER[] = "If-None-Match"; WebServer::WebServer(IPAddress addr, int port) @@ -393,6 +394,11 @@ void WebServer::enableCrossOrigin(boolean value) { enableCORS(value); } +void WebServer::enableETag(bool enable, ETagFunction fn) { + _eTagEnabled = enable; + _eTagFunction = fn; +} + void WebServer::_prepareHeader(String& response, int code, const char* content_type, size_t contentLength) { response = String(F("HTTP/1.")) + String(_currentVersion) + ' '; response += String(code); @@ -597,11 +603,12 @@ String WebServer::header(String name) { } void WebServer::collectHeaders(const char* headerKeys[], const size_t headerKeysCount) { - _headerKeysCount = headerKeysCount + 1; + _headerKeysCount = headerKeysCount + 2; if (_currentHeaders) delete[]_currentHeaders; _currentHeaders = new RequestArgument[_headerKeysCount]; _currentHeaders[0].key = FPSTR(AUTHORIZATION_HEADER); + _currentHeaders[1].key = FPSTR(ETAG_HEADER); for (int i = 1; i < _headerKeysCount; i++){ _currentHeaders[i].key = headerKeys[i-1]; } diff --git a/libraries/WebServer/src/WebServer.h b/libraries/WebServer/src/WebServer.h index fc60d16496f..af307b2e434 100644 --- a/libraries/WebServer/src/WebServer.h +++ b/libraries/WebServer/src/WebServer.h @@ -27,6 +27,7 @@ #include #include #include +#include #include "HTTP_Method.h" #include "Uri.h" @@ -130,6 +131,8 @@ class WebServer void enableDelay(boolean value); void enableCORS(boolean value = true); void enableCrossOrigin(boolean value = true); + typedef std::function ETagFunction; + void enableETag(bool enable, ETagFunction fn = nullptr); void setContentLength(const size_t contentLength); void sendHeader(const String& name, const String& value, bool first = false); @@ -146,6 +149,9 @@ class WebServer return _currentClient.write(file); } + bool _eTagEnabled = false; + ETagFunction _eTagFunction = nullptr; + protected: virtual size_t _currentClientWrite(const char* b, size_t l) { return _currentClient.write( b, l ); } virtual size_t _currentClientWrite_P(PGM_P b, size_t l) { return _currentClient.write_P( b, l ); } diff --git a/libraries/WebServer/src/detail/RequestHandlersImpl.h b/libraries/WebServer/src/detail/RequestHandlersImpl.h index 4a7c28e58ae..d10bd19d12b 100644 --- a/libraries/WebServer/src/detail/RequestHandlersImpl.h +++ b/libraries/WebServer/src/detail/RequestHandlersImpl.h @@ -5,6 +5,8 @@ #include "mimetable.h" #include "WString.h" #include "Uri.h" +#include +#include using namespace mime; @@ -91,6 +93,7 @@ class StaticRequestHandler : public RequestHandler { log_v("StaticRequestHandler::handle: request=%s _uri=%s\r\n", requestUri.c_str(), _uri.c_str()); String path(_path); + String eTagCode; if (!_isFile) { // Base URI doesn't point to a file. @@ -117,9 +120,26 @@ class StaticRequestHandler : public RequestHandler { if (!f || !f.available()) return false; + if (server._eTagEnabled) { + if (server._eTagFunction) { + eTagCode = (server._eTagFunction)(_fs, path); + } else { + eTagCode = calcETag(_fs, path); + } + + if (server.header("If-None-Match") == eTagCode) { + server.send(304); + return true; + } + } + if (_cache_header.length() != 0) server.sendHeader("Cache-Control", _cache_header); + if ((server._eTagEnabled) && (eTagCode.length() > 0)) { + server.sendHeader("ETag", eTagCode); + } + server.streamFile(f, contentType); return true; } @@ -139,6 +159,26 @@ class StaticRequestHandler : public RequestHandler { return String(buff); } + // calculate an ETag for a file in filesystem based on md5 checksum + // that can be used in the http headers - include quotes. + static String calcETag(FS &fs, const String &path) { + String result; + + // calculate eTag using md5 checksum + uint8_t md5_buf[16]; + File f = fs.open(path, "r"); + MD5Builder calcMD5; + calcMD5.begin(); + calcMD5.addStream(f, f.size()); + calcMD5.calculate(); + calcMD5.getBytes(md5_buf); + f.close(); + // create a minimal-length eTag using base64 byte[]->text encoding. + result = "\"" + base64::encode(md5_buf, 16) + "\""; + return(result); + } // calcETag + + protected: FS _fs; String _uri; From d27bbd79038fa3bcca8eef76456d56997c2789c4 Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Mon, 9 Jan 2023 09:23:12 +0100 Subject: [PATCH 02/10] Supporting ETag http headers on static files --- libraries/WebServer/src/WebServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index fc1cad571fa..47d88df61d7 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -609,8 +609,8 @@ void WebServer::collectHeaders(const char* headerKeys[], const size_t headerKeys _currentHeaders = new RequestArgument[_headerKeysCount]; _currentHeaders[0].key = FPSTR(AUTHORIZATION_HEADER); _currentHeaders[1].key = FPSTR(ETAG_HEADER); - for (int i = 1; i < _headerKeysCount; i++){ - _currentHeaders[i].key = headerKeys[i-1]; + for (int i = 2; i < _headerKeysCount; i++){ + _currentHeaders[i].key = headerKeys[i-2]; } } From ae8a55b2af41e86aff7ac65853a4068fca418e61 Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Mon, 9 Jan 2023 17:44:00 +0100 Subject: [PATCH 03/10] WebServer Example and Doku --- .../WebServer/examples/WebServer/README.md | 216 +++++++++++++ .../examples/WebServer/WebServer.ino | 296 ++++++++++++++++++ .../examples/WebServer/builtinfiles.h | 63 ++++ .../examples/WebServer/data/files.htm | 65 ++++ .../examples/WebServer/data/index.htm | 25 ++ .../examples/WebServer/data/style.css | 10 + .../WebServer/examples/WebServer/secrets.h | 13 + 7 files changed, 688 insertions(+) create mode 100644 libraries/WebServer/examples/WebServer/README.md create mode 100644 libraries/WebServer/examples/WebServer/WebServer.ino create mode 100644 libraries/WebServer/examples/WebServer/builtinfiles.h create mode 100644 libraries/WebServer/examples/WebServer/data/files.htm create mode 100644 libraries/WebServer/examples/WebServer/data/index.htm create mode 100644 libraries/WebServer/examples/WebServer/data/style.css create mode 100644 libraries/WebServer/examples/WebServer/secrets.h diff --git a/libraries/WebServer/examples/WebServer/README.md b/libraries/WebServer/examples/WebServer/README.md new file mode 100644 index 00000000000..cb05d4fe9f0 --- /dev/null +++ b/libraries/WebServer/examples/WebServer/README.md @@ -0,0 +1,216 @@ +# WebServer example documentation and hints + +This example shows different techniques on how to use and extend the WebServer for specific purposes + +It is a small project in it's own and has some files to use on the web server to show how to use simple REST based services. + +It requires some space for a filesystem and runs fine boards with 4 MByte flash using the following options: + +* Board: ESP32 Dev Module +* Partition Scheme: Default 4MB with spiffs (1.2MB APP/1.5MB SPIFFS) + but LittleFS will be used in the partition (not SPIFFS) + +It features + +* Setup a web server +* redirect when accessing the url with servername only +* get real time by using builtin NTP functionality +* send HTML responses from Sketch (see builtinfiles.h) +* use a LittleFS file system on the data partition for static files +* use http ETag Header for client side caching of static files +* use custom ETag calculation for static files +* extended FileServerHandler for uploading and deleting static files +* uploading files using drag & drop +* serve APIs using REST services (/api/list, /api/sysinfo) +* define HTML response when no file/api/handler was found + +## Use the Example + +* In the file `secrets.h` you can add the home WiFi network name ans password. +* Compile and upload to the device. +* Have a look into the monitoring output. +* Open or using a browser. +* You will be redirected to as there are no files yet in the file system. +* Drag the files from the data folder onto the drop area shown in the browser. +* See below for more details + +## Implementing a web server + +The WebServer library offers a simple path to implement a web server on a ESP32 board. + +The advantage on using the WebServer instead of the plain simple WiFiServer is that the WebServer +takes much care about the http protocol conventions and features and allows easily access to parameters. +It offers plug-in capabilities by registering specific functionalities that will be outlined below. + +### Initialization + +In the setup() function in the webserver.ino sketch file the following steps are implemented to make the webserver available on the local network. + +* Create a webserver listening to port 80 for http requests. +* Initialize the access to the filesystem in the free flash memory. +* Connect to the local WiFi network. Here is only a straight-forward implementation hard-coding network name and passphrase. You may consider to use something like the WiFiManager library in real applications. +* Register the device in DNS using a known hostname. +* Registering several plug-ins (see below). +* Starting the web server. + +### Running + +In the loop() function the web server will be given time to receive and send network packages by calling +`server.handleClient();`. + +## Registering simple functions to implement RESTful services + +Registering function is the simplest integration mechanism available to add functionality. The server offers the `on(path, function)` methods that take the URL and the function as parameters. + +There are 2 functions implemented that get registered to handle incoming GET requests for given URLs. + +The JSON data format is used often for such services as it is the "natural" data format of the browser using javascript. + +When the **handleSysInfo()** function is registered and a browser requests for the function will be called and can collect the requested information. + +> ```CPP +> server.on("/api/sysinfo", handleSysInfo); +> ``` + +The result in this case is a JSON object that is assembled in the result String variable and the returned as a response to the client also giving the information about the data format. + +You can try this request in a browser by opening in the address bar. + +> ```CPP +> server.on("/api/sysinfo", handleList); +> ``` + +The function **handleList()** is registered the same way to return the list of files in the file system also returning a JSON object including name, size and the last modification timestamp. + +You can try this request in a browser by opening in the address bar. + +## Registering a function to send out some static content from a String + +This is an example of registering a inline function in the web server. +The 2. parameter of the on() method is a so called CPP lamda function (without a name) +that actually has only one line of functionality by sending a string as result to the client. + +> ``` cpp +> server.on("/$upload.htm", []() { +> server.send(200, "text/html", FPSTR(uploadContent)); +> }); +> ``` + +Here the text from a static String with html code is returned instead of a file from the filesystem. +The content of this string can be found in the file `builtinfiles.h`. It contains a small html+javascript implementation +that allows uploading new files into the empty filesystem. + +Just open and drag some files from the data folder on the drop area. + +## Registering a function to handle requests to the server without a path + +Often servers are addressed by using the base URL like where no further path details is given. +Of course we like the user to be redirected to something usable. Therefore the `handleRoot()` function is registered: + +> ``` cpp +> server.on("/$upload.htm", handleRoot); +> ``` + +The `handleRoot()` function checks the filesystem for the file named **/index.htm** and creates a redirect to this file when the file exists. +Otherwise the redirection goes to the built-in **/$upload.htm** web page. + +## Using the serveStatic plug-in + +The **serveStatic** plug in is part of the library and handles delivering files from the filesystem to the client. It can be customized in some ways. + +> ``` cpp +> server.enableCORS(true); +> server.enableETag(true); +> server.serveStatic("/", LittleFS, "/"); +> ``` + +### Cross-Origin Ressource Sharing (CORS) + +The `enableCORS(true)` function adds a `Access-Control-Allow-Origin: *` http-header to all responses to the client +to inform that it is allowed to call URLs and services on this server from other web sites. + +The feature is disabled by default (in the current version) and when you like to disable this then you should call `enableCORS(false)` during setup. + +* Web sites providing high sensitive information like online banking this is disabled most of the times. +* Web sites providing advertising information or reusable scripts / images this is enabled. + +### enabling ETag support + +To enable this in the embedded web server the `enableETag()` can be used. +(next to enableCORS) + +In the simplest version just call `enableETag(true)` to enable the internal ETag generation that calcs the hint using a md5 checksum in base64 encoded form. This is an simple approach that adds some time for calculation on every request but avoids network traffic. + +The headers will look like: + +``` txt +If-None-Match: "GhZka3HevoaEBbtQOgOqlA==" +ETag: "GhZka3HevoaEBbtQOgOqlA==" +``` + + +### ETag support customization + +The enableETag() function has an optional second optional parameter to provide a function for ETag calculation of files. + +The function enables eTags for all files by using calculating a value from the last write timestamp: + +``` cpp +server.enableETag(true, [](FS &fs, const String &path) -> String { + File f = fs.open(path, "r"); + String eTag = String(f.getLastWrite(), 16); // use file modification timestamp to create ETag + f.close(); + return (eTag); +}); +``` + +The headers will look like: + +``` txt +ETag: "63bbaeb5" +If-None-Match: "63bbaeb5" +``` + + +## Registering a full-featured handler as plug-in + +The example also implements the class `FileServerHandler` derived from the class `RequestHandler` to plug in functionality +that can handle more complex requests without giving a fixed URL. +It implements uploading and deleting files in the file system that is not implemented by the standard server.serveStatic functionality. + +This class has to implements several functions and works in a more detailed way: + +* The `canHandle()` method can inspect the given http method and url to decide weather the RequestFileHandler can handle the incoming request or not. + + In this case the RequestFileHandler will return true when the request method is an POST for upload or a DELETE for deleting files. + + The regular GET requests will be ignored and therefore handled by the also registered server.serveStatic handler. + +* The function `handle()` then implements the real deletion of the file. + +* The `canUpload()`and `upload()` methods work similar while the `upload()` method is called multiple times to create, append data and close the new file. + + +## Registering a special handler for "file not found" + +Any other incoming request that was not handled by the registered plug-ins above can be detected by registering + +> ``` cpp +> // handle cases when file is not found +> server.onNotFound([]() { +> // standard not found in browser. +> server.send(404, "text/html", FPSTR(notFoundContent)); +> }); +> ``` + +This allows sending back an "friendly" result for the browser. Here a simple html page is created from a static string. +You can easily change the html code in the file `builtinfiles.h`. + +## customizations + +You may like to change the hostname and the timezone in the lines: + +> ``` cpp +> #define HOSTNAME "webserver" +> #define TIMEZONE "CET-1CEST,M3.5.0,M10.5.0/3" +> ``` diff --git a/libraries/WebServer/examples/WebServer/WebServer.ino b/libraries/WebServer/examples/WebServer/WebServer.ino new file mode 100644 index 00000000000..c38ee334f1a --- /dev/null +++ b/libraries/WebServer/examples/WebServer/WebServer.ino @@ -0,0 +1,296 @@ +// @file WebServer.ino +// @brief Example WebServer implementation using the ESP32 WebServer +// and most common use cases related to web servers. +// +// * Setup a web server +// * redirect when accessing the url with servername only +// * get real time by using builtin NTP functionality +// * send HTML responses from Sketch (see builtinfiles.h) +// * use a LittleFS file system on the data partition for static files +// * use http ETag Header for client side caching of static files +// * use custom ETag calculation for static files +// * extended FileServerHandler for uploading and deleting static files +// * extended FileServerHandler for uploading and deleting static files +// * serve APIs using REST services (/api/list, /api/sysinfo) +// * define HTML response when no file/api/handler was found +// +// See also README.md for instructions and hints. +// +// Please use the following Arduino IDE configuration +// +// * Board: ESP32 Dev Module +// * Partition Scheme: Default 4MB with spiffs (1.2MB APP/1.5MB SPIFFS) +// but LittleFS will be used in the partition (not SPIFFS) +// * other setting as applicable +// +// Changelog: +// 21.07.2021 creation, first version +// 08.01.2023 ESP32 version with ETag + +#include +#include + +#include "secrets.h" // add WLAN Credentials in here. + +#include // File System for Web Server Files +#include // This file system is used. + +// mark parameters not used in example +#define UNUSED __attribute__((unused)) + +// TRACE output simplified, can be deactivated here +#define TRACE(...) Serial.printf(__VA_ARGS__) + +// name of the server. You reach it using http://webserver +#define HOSTNAME "webserver" + +// local time zone definition (Berlin) +#define TIMEZONE "CET-1CEST,M3.5.0,M10.5.0/3" + +// need a WebServer for http access on port 80. +WebServer server(80); + +// The text of builtin files are in this header file +#include "builtinfiles.h" + +// enable the CUSTOM_ETAG_CALC to enable calculation of ETags by a custom function +#define CUSTOM_ETAG_CALC + +// ===== Simple functions used to answer simple GET requests ===== + +// This function is called when the WebServer was requested without giving a filename. +// This will redirect to the file index.htm when it is existing otherwise to the built-in $upload.htm page +void handleRedirect() { + TRACE("Redirect...\n"); + String url = "/index.htm"; + + if (!LittleFS.exists(url)) { url = "/$upload.htm"; } + + server.sendHeader("Location", url, true); + server.send(302); +} // handleRedirect() + + +// This function is called when the WebServer was requested to list all existing files in the filesystem. +// a JSON array with file information is returned. +void handleListFiles() { + File dir = LittleFS.open("/", "r"); + String result; + + result += "[\n"; + while (File entry = dir.openNextFile()) { + if (result.length() > 4) { result += ",\n"; } + result += " {"; + result += "\"type\": \"file\", "; + result += "\"name\": \"" + String(entry.name()) + "\", "; + result += "\"size\": " + String(entry.size()) + ", "; + result += "\"time\": " + String(entry.getLastWrite()); + result += "}"; + } // while + + result += "\n]"; + server.sendHeader("Cache-Control", "no-cache"); + server.send(200, "text/javascript; charset=utf-8", result); +} // handleListFiles() + + +// This function is called when the sysInfo service was requested. +void handleSysInfo() { + String result; + + result += "{\n"; + result += " \"Chip Model\": " + String(ESP.getChipModel()) + ",\n"; + result += " \"Chip Cores\": " + String(ESP.getChipCores()) + ",\n"; + result += " \"Chip Revision\": " + String(ESP.getChipRevision()) + ",\n"; + result += " \"flashSize\": " + String(ESP.getFlashChipSize()) + ",\n"; + result += " \"freeHeap\": " + String(ESP.getFreeHeap()) + ",\n"; + result += " \"fsTotalBytes\": " + String(LittleFS.totalBytes()) + ",\n"; + result += " \"fsUsedBytes\": " + String(LittleFS.usedBytes()) + ",\n"; + result += "}"; + + server.sendHeader("Cache-Control", "no-cache"); + server.send(200, "text/javascript; charset=utf-8", result); +} // handleSysInfo() + + +// ===== Request Handler class used to answer more complex requests ===== + +// The FileServerHandler is registered to the web server to support DELETE and UPLOAD of files into the filesystem. +class FileServerHandler : public RequestHandler { +public: + // @brief Construct a new File Server Handler object + // @param fs The file system to be used. + // @param path Path to the root folder in the file system that is used for serving static data down and upload. + // @param cache_header Cache Header to be used in replies. + FileServerHandler() { + TRACE("FileServerHandler is registered\n"); + } + + + // @brief check incoming request. Can handle POST for uploads and DELETE. + // @param requestMethod method of the http request line. + // @param requestUri request ressource from the http request line. + // @return true when method can be handled. + bool canHandle(HTTPMethod requestMethod, String UNUSED uri) override { + return ((requestMethod == HTTP_POST) || (requestMethod == HTTP_DELETE)); + } // canHandle() + + + bool canUpload(String uri) override { + // only allow upload on root fs level. + return (uri == "/"); + } // canUpload() + + + bool handle(WebServer &server, HTTPMethod requestMethod, String requestUri) override { + // ensure that filename starts with '/' + String fName = requestUri; + if (!fName.startsWith("/")) { fName = "/" + fName; } + + if (requestMethod == HTTP_POST) { + // all done in upload. no other forms. + + } else if (requestMethod == HTTP_DELETE) { + if (LittleFS.exists(fName)) { LittleFS.remove(fName); } + } // if + + server.send(200); // all done. + return (true); + } // handle() + + + // uploading process + void upload(WebServer UNUSED &server, String UNUSED _requestUri, HTTPUpload &upload) override { + // ensure that filename starts with '/' + String fName = upload.filename; + if (!fName.startsWith("/")) { fName = "/" + fName; } + + if (upload.status == UPLOAD_FILE_START) { + // Open the file + if (LittleFS.exists(fName)) { LittleFS.remove(fName); } // if + _fsUploadFile = LittleFS.open(fName, "w"); + + } else if (upload.status == UPLOAD_FILE_WRITE) { + // Write received bytes + if (_fsUploadFile) { _fsUploadFile.write(upload.buf, upload.currentSize); } + + } else if (upload.status == UPLOAD_FILE_END) { + // Close the file + if (_fsUploadFile) { _fsUploadFile.close(); } + } // if + } // upload() + +protected: + File _fsUploadFile; +}; + + +// Setup everything to make the webserver work. +void setup(void) { + delay(3000); // wait for serial monitor to start completely. + + // Use Serial port for some trace information from the example + Serial.begin(115200); + Serial.setDebugOutput(false); + + TRACE("Starting WebServer example...\n"); + + TRACE("Mounting the filesystem...\n"); + if (!LittleFS.begin()) { + TRACE("could not mount the filesystem...\n"); + delay(2000); + TRACE("formatting...\n"); + LittleFS.format(); + delay(2000); + TRACE("restart.\n"); + delay(2000); + ESP.restart(); + } + + // allow to address the device by the given name e.g. http://webserver + WiFi.setHostname(HOSTNAME); + + // start WiFI + WiFi.mode(WIFI_STA); + if (strlen(ssid) == 0) { + WiFi.begin(); + } else { + WiFi.begin(ssid, passPhrase); + } + + TRACE("Connect to WiFi...\n"); + while (WiFi.status() != WL_CONNECTED) { + delay(500); + TRACE("."); + } + TRACE("connected.\n"); + + // Ask for the current time using NTP request builtin into ESP firmware. + TRACE("Setup ntp...\n"); + configTzTime(TIMEZONE, "pool.ntp.org"); + + TRACE("Register redirect...\n"); + + // register a redirect handler when only domain name is given. + server.on("/", HTTP_GET, handleRedirect); + + TRACE("Register service handlers...\n"); + + // serve a built-in htm page + server.on("/$upload.htm", []() { + server.send(200, "text/html", FPSTR(uploadContent)); + }); + + // register some REST services + server.on("/api/list", HTTP_GET, handleListFiles); + server.on("/api/sysinfo", HTTP_GET, handleSysInfo); + + TRACE("Register file system handlers...\n"); + + // UPLOAD and DELETE of files in the file system using a request handler. + server.addHandler(new FileServerHandler()); + + // // enable CORS header in webserver results + server.enableCORS(true); + + // enable ETAG header in webserver results (used by serveStatic handler) +#if defined(CUSTOM_ETAG_CALC) + // This is a fast custom eTag generator. It returns a value based on the time the file was updated like + // ETag: 63bbceb5 + server.enableETag(true, [](FS &fs, const String &path) -> String { + File f = fs.open(path, "r"); + String eTag = String(f.getLastWrite(), 16); // use file modification timestamp to create ETag + f.close(); + return (eTag); + }); + +#else + // enable standard ETAG calculation using md5 checksum of file content. + server.enableETag(true); +#endif + + // serve all static files + server.serveStatic("/", LittleFS, "/"); + + TRACE("Register default (not found) answer...\n"); + + // handle cases when file is not found + server.onNotFound([]() { + // standard not found in browser. + server.send(404, "text/html", FPSTR(notFoundContent)); + }); + + server.begin(); + + TRACE("open or \n", + WiFi.getHostname(), + WiFi.localIP().toString().c_str()); +} // setup + + +// run the server... +void loop(void) { + server.handleClient(); +} // loop() + +// end. diff --git a/libraries/WebServer/examples/WebServer/builtinfiles.h b/libraries/WebServer/examples/WebServer/builtinfiles.h new file mode 100644 index 00000000000..210b18c1a58 --- /dev/null +++ b/libraries/WebServer/examples/WebServer/builtinfiles.h @@ -0,0 +1,63 @@ +/** + * @file builtinfiles.h + * @brief This file is part of the WebServer example for the ESP8266WebServer. + * + * This file contains long, multiline text variables for all builtin resources. + */ + +// used for $upload.htm +static const char uploadContent[] PROGMEM = +R"==( + + + + + + + Upload + + + +

Upload

+ +
+
Drop files here...
+ + + +)=="; + +// used for $upload.htm +static const char notFoundContent[] PROGMEM = R"==( + + + Ressource not found + + +

The ressource was not found.

+

Start again

+ +)=="; diff --git a/libraries/WebServer/examples/WebServer/data/files.htm b/libraries/WebServer/examples/WebServer/data/files.htm new file mode 100644 index 00000000000..026dd9ce5c7 --- /dev/null +++ b/libraries/WebServer/examples/WebServer/data/files.htm @@ -0,0 +1,65 @@ + + + + Files + + + + +

Files on Server

+ +

These files are available on the server to be opened or delete:

+
+
+ + + + + \ No newline at end of file diff --git a/libraries/WebServer/examples/WebServer/data/index.htm b/libraries/WebServer/examples/WebServer/data/index.htm new file mode 100644 index 00000000000..58a04ede5be --- /dev/null +++ b/libraries/WebServer/examples/WebServer/data/index.htm @@ -0,0 +1,25 @@ + + + + HomePage + + + + +

Homepage of the WebServer Example

+ +

The following pages are available:

+ + +

The following REST services are available:

+ + + \ No newline at end of file diff --git a/libraries/WebServer/examples/WebServer/data/style.css b/libraries/WebServer/examples/WebServer/data/style.css new file mode 100644 index 00000000000..95ac48e727a --- /dev/null +++ b/libraries/WebServer/examples/WebServer/data/style.css @@ -0,0 +1,10 @@ +html, body { + color: #111111; font-family: Arial, ui-sans-serif, sans-serif; font-size: 1em; background-color: #f0f0f0; +} + +#list > div { + margin: 0 0 0.5rem 0; +} + +a { color: inherit; cursor: pointer; } + diff --git a/libraries/WebServer/examples/WebServer/secrets.h b/libraries/WebServer/examples/WebServer/secrets.h new file mode 100644 index 00000000000..0585287d3e7 --- /dev/null +++ b/libraries/WebServer/examples/WebServer/secrets.h @@ -0,0 +1,13 @@ +// Secrets for your local home network + +// This is a "hard way" to configure your local WiFi network name and passphrase +// into the source code and the uploaded sketch. +// +// Using the WiFi Manager is preferred and avoids reprogramming when your network changes. +// See https://homeding.github.io/#page=/wifimanager.md + +// ssid and passPhrase can be used when compiling for a specific environment as a 2. option. + +// add you wifi network name and PassPhrase or use WiFi Manager +const char *ssid = ""; +const char *passPhrase = ""; From af6c27b3333a05240218199c702a0fc781b6108b Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Wed, 1 Feb 2023 21:44:44 +0100 Subject: [PATCH 04/10] new template for readme added. --- .../WebServer/examples/WebServer/README.md | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/libraries/WebServer/examples/WebServer/README.md b/libraries/WebServer/examples/WebServer/README.md index cb05d4fe9f0..b5465098057 100644 --- a/libraries/WebServer/examples/WebServer/README.md +++ b/libraries/WebServer/examples/WebServer/README.md @@ -1,10 +1,10 @@ -# WebServer example documentation and hints +# Arduino-ESP32 WebServer Example for WebServer Library This example shows different techniques on how to use and extend the WebServer for specific purposes It is a small project in it's own and has some files to use on the web server to show how to use simple REST based services. -It requires some space for a filesystem and runs fine boards with 4 MByte flash using the following options: +This example requires some space for a filesystem and runs fine boards with 4 MByte flash using the following options: * Board: ESP32 Dev Module * Partition Scheme: Default 4MB with spiffs (1.2MB APP/1.5MB SPIFFS) @@ -24,8 +24,18 @@ It features * serve APIs using REST services (/api/list, /api/sysinfo) * define HTML response when no file/api/handler was found +## Supported Targets + +Currently, this example supports the following targets. + +| Supported Targets | ESP32 | ESP32-S2 | ESP32-C3 | +| ----------------- | ----- | -------- | -------- | +| | yes | yes | yes | + ## Use the Example +How to install the Arduino IDE: [Install Arduino IDE](https://github.com/espressif/arduino-esp32/tree/master/docs/arduino-ide). + * In the file `secrets.h` you can add the home WiFi network name ans password. * Compile and upload to the device. * Have a look into the monitoring output. @@ -36,7 +46,7 @@ It features ## Implementing a web server -The WebServer library offers a simple path to implement a web server on a ESP32 board. +The WebServer library offers a simple path to implement a web server on a ESP32 based board. The advantage on using the WebServer instead of the plain simple WiFiServer is that the WebServer takes much care about the http protocol conventions and features and allows easily access to parameters. @@ -190,7 +200,6 @@ This class has to implements several functions and works in a more detailed way: * The `canUpload()`and `upload()` methods work similar while the `upload()` method is called multiple times to create, append data and close the new file. - ## Registering a special handler for "file not found" Any other incoming request that was not handled by the registered plug-ins above can be detected by registering @@ -214,3 +223,24 @@ You may like to change the hostname and the timezone in the lines: > #define HOSTNAME "webserver" > #define TIMEZONE "CET-1CEST,M3.5.0,M10.5.0/3" > ``` + +## Troubleshooting + +Have a look in the Serial output for some additional runtime information. + +## Contribute + +To know how to contribute to this project, see [How to contribute.](https://github.com/espressif/arduino-esp32/blob/master/CONTRIBUTING.rst) + +If you have any **feedback** or **issue** to report on this example/library, please open an issue or fix it by creating a new PR. Contributions are more than welcome! + +Before creating a new issue, be sure to try Troubleshooting and check if the same issue was already created by someone else. + +## Resources + +* Official ESP32 Forum: [Link](https://esp32.com) +* Arduino-ESP32 Official Repository: [espressif/arduino-esp32](https://github.com/espressif/arduino-esp32) +* ESP32 Datasheet: [Link to datasheet](https://www.espressif.com/sites/default/files/documentation/esp32_datasheet_en.pdf) +* ESP32-S2 Datasheet: [Link to datasheet](https://www.espressif.com/sites/default/files/documentation/esp32-s2_datasheet_en.pdf) +* ESP32-C3 Datasheet: [Link to datasheet](https://www.espressif.com/sites/default/files/documentation/esp32-c3_datasheet_en.pdf) +* Official ESP-IDF documentation: [ESP-IDF](https://idf.espressif.com) From 108ea5a823996615732f01e4662fd6951c957c7e Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Wed, 1 Feb 2023 22:01:38 +0100 Subject: [PATCH 05/10] example updated, som more TRACE output. --- .../examples/WebServer/WebServer.ino | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/libraries/WebServer/examples/WebServer/WebServer.ino b/libraries/WebServer/examples/WebServer/WebServer.ino index c38ee334f1a..b2f28533f24 100644 --- a/libraries/WebServer/examples/WebServer/WebServer.ino +++ b/libraries/WebServer/examples/WebServer/WebServer.ino @@ -163,20 +163,31 @@ public: void upload(WebServer UNUSED &server, String UNUSED _requestUri, HTTPUpload &upload) override { // ensure that filename starts with '/' String fName = upload.filename; + static size_t uploadSize; if (!fName.startsWith("/")) { fName = "/" + fName; } if (upload.status == UPLOAD_FILE_START) { - // Open the file - if (LittleFS.exists(fName)) { LittleFS.remove(fName); } // if + // Open the file for writing + TRACE("starting upload file %s...", fName.c_str()); + if (LittleFS.exists(fName)) { + LittleFS.remove(fName); + } // if _fsUploadFile = LittleFS.open(fName, "w"); + uploadSize = 0; } else if (upload.status == UPLOAD_FILE_WRITE) { // Write received bytes - if (_fsUploadFile) { _fsUploadFile.write(upload.buf, upload.currentSize); } + if (_fsUploadFile) { + _fsUploadFile.write(upload.buf, upload.currentSize); + uploadSize += upload.currentSize; + } } else if (upload.status == UPLOAD_FILE_END) { // Close the file - if (_fsUploadFile) { _fsUploadFile.close(); } + if (_fsUploadFile) { + _fsUploadFile.close(); + } + TRACE("uploading %d bytes done.", uploadSize); } // if } // upload() From 3247e1f22b3013af2fb3273da674df8703f95da7 Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Wed, 1 Feb 2023 22:09:48 +0100 Subject: [PATCH 06/10] better TRACE formatting. --- libraries/WebServer/examples/WebServer/WebServer.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/WebServer/examples/WebServer/WebServer.ino b/libraries/WebServer/examples/WebServer/WebServer.ino index b2f28533f24..e8ee1eba5c8 100644 --- a/libraries/WebServer/examples/WebServer/WebServer.ino +++ b/libraries/WebServer/examples/WebServer/WebServer.ino @@ -168,7 +168,7 @@ public: if (upload.status == UPLOAD_FILE_START) { // Open the file for writing - TRACE("starting upload file %s...", fName.c_str()); + TRACE("starting upload file %s...\n", fName.c_str()); if (LittleFS.exists(fName)) { LittleFS.remove(fName); } // if @@ -187,7 +187,7 @@ public: if (_fsUploadFile) { _fsUploadFile.close(); } - TRACE("uploading %d bytes done.", uploadSize); + TRACE("uploading %d bytes done.\n", uploadSize); } // if } // upload() From f0d5e5561683e8b0fdb075b62a94f9197d8c9c1c Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Fri, 3 Feb 2023 18:33:19 +0100 Subject: [PATCH 07/10] upload and trace enhancements --- .../WebServer/examples/WebServer/README.md | 38 +++++++++++++++++ .../examples/WebServer/WebServer.ino | 42 +++++++++++++++---- .../examples/WebServer/data/files.htm | 2 +- 3 files changed, 72 insertions(+), 10 deletions(-) diff --git a/libraries/WebServer/examples/WebServer/README.md b/libraries/WebServer/examples/WebServer/README.md index b5465098057..09d1cac7459 100644 --- a/libraries/WebServer/examples/WebServer/README.md +++ b/libraries/WebServer/examples/WebServer/README.md @@ -200,6 +200,44 @@ This class has to implements several functions and works in a more detailed way: * The `canUpload()`and `upload()` methods work similar while the `upload()` method is called multiple times to create, append data and close the new file. +## File upload + +By opening you can easily upload files by dragging them over the drop area. + +Just take the files from the data folder to create some files that can explore the server functionality. + +Files will be uploaded to the root folder of the file system. and you will see it next time using . + +The filesize that is uploaded is not known when the upload mechanism in function +FileServerHandler::upload gets started. + +Uploading a file that fits into the available filesystem space +can be found in the Serial output: + +``` txt +starting upload file /file.txt... +finished. +1652 bytes uploaded. +``` + +Uploading a file that doesn't fit can be detected while uploading when writing to the filesystem fails. +However upload cannot be aborted by the current handler implementation. + +The solution implemented here is to delete the partially uploaded file and wait for the upload ending. +The following can be found in the Serial output: + +``` txt +starting upload file /huge.jpg... +./components/esp_littlefs/src/littlefs/lfs.c:584:error: No more free space 531 + write error! +finished. +``` + +You can see on the Serial output that one filesystem write error is reported. + +Please be patient and wait for the upload ending even when writing to the filesystem is disabled +it maybe take more than a minute. + ## Registering a special handler for "file not found" Any other incoming request that was not handled by the registered plug-ins above can be detected by registering diff --git a/libraries/WebServer/examples/WebServer/WebServer.ino b/libraries/WebServer/examples/WebServer/WebServer.ino index e8ee1eba5c8..1193a2e6114 100644 --- a/libraries/WebServer/examples/WebServer/WebServer.ino +++ b/libraries/WebServer/examples/WebServer/WebServer.ino @@ -147,11 +147,16 @@ public: String fName = requestUri; if (!fName.startsWith("/")) { fName = "/" + fName; } + TRACE("handle %s\n", fName.c_str()); + if (requestMethod == HTTP_POST) { // all done in upload. no other forms. } else if (requestMethod == HTTP_DELETE) { - if (LittleFS.exists(fName)) { LittleFS.remove(fName); } + if (LittleFS.exists(fName)) { + TRACE("DELETE %s\n", fName.c_str()); + LittleFS.remove(fName); + } } // if server.send(200); // all done. @@ -160,15 +165,18 @@ public: // uploading process - void upload(WebServer UNUSED &server, String UNUSED _requestUri, HTTPUpload &upload) override { + void + upload(WebServer UNUSED &server, String UNUSED _requestUri, HTTPUpload &upload) override { // ensure that filename starts with '/' - String fName = upload.filename; static size_t uploadSize; - if (!fName.startsWith("/")) { fName = "/" + fName; } if (upload.status == UPLOAD_FILE_START) { + String fName = upload.filename; + // Open the file for writing - TRACE("starting upload file %s...\n", fName.c_str()); + if (!fName.startsWith("/")) { fName = "/" + fName; } + TRACE("start uploading file %s...\n", fName.c_str()); + if (LittleFS.exists(fName)) { LittleFS.remove(fName); } // if @@ -178,18 +186,34 @@ public: } else if (upload.status == UPLOAD_FILE_WRITE) { // Write received bytes if (_fsUploadFile) { - _fsUploadFile.write(upload.buf, upload.currentSize); + size_t written = _fsUploadFile.write(upload.buf, upload.currentSize); + if (written < upload.currentSize) { + // upload failed + TRACE(" write error!\n"); + _fsUploadFile.close(); + + // delete file to free up space in filesystem + String fName = upload.filename; + if (!fName.startsWith("/")) { fName = "/" + fName; } + LittleFS.remove(fName); + } uploadSize += upload.currentSize; - } + // TRACE("free:: %d of %d\n", LittleFS.usedBytes(), LittleFS.totalBytes()); + // TRACE("written:: %d of %d\n", written, upload.currentSize); + // TRACE("totalSize: %d\n", upload.currentSize + upload.totalSize); + } // if } else if (upload.status == UPLOAD_FILE_END) { + TRACE("finished.\n"); // Close the file if (_fsUploadFile) { _fsUploadFile.close(); + TRACE(" %d bytes uploaded.\n", upload.totalSize); } - TRACE("uploading %d bytes done.\n", uploadSize); } // if - } // upload() + + } // upload() + protected: File _fsUploadFile; diff --git a/libraries/WebServer/examples/WebServer/data/files.htm b/libraries/WebServer/examples/WebServer/data/files.htm index 026dd9ce5c7..4e8804154ac 100644 --- a/libraries/WebServer/examples/WebServer/data/files.htm +++ b/libraries/WebServer/examples/WebServer/data/files.htm @@ -51,7 +51,7 @@

Files on Server

var t = evt.target; if (t.className === 'deleteFile') { var fname = t.parentElement.innerText; - fname = '/'+ fname.split(' ')[0]; + fname = fname.split(' ')[0]; if (window.confirm("Delete " + fname + " ?")) { fetch(fname, { method: 'DELETE' }); document.location.reload(false); From ea1389b23cb28a7a052a355dd1b97cb6bfaabdbc Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Wed, 29 Nov 2023 19:59:09 +0100 Subject: [PATCH 08/10] Create .skip.esp32h2 --- libraries/WebServer/examples/WebServer/.skip.esp32h2 | 1 + 1 file changed, 1 insertion(+) create mode 100644 libraries/WebServer/examples/WebServer/.skip.esp32h2 diff --git a/libraries/WebServer/examples/WebServer/.skip.esp32h2 b/libraries/WebServer/examples/WebServer/.skip.esp32h2 new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/libraries/WebServer/examples/WebServer/.skip.esp32h2 @@ -0,0 +1 @@ + From e39192730ba17ffa94130a1d6bd8e1b85bf8a5ea Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Wed, 29 Nov 2023 20:06:00 +0100 Subject: [PATCH 09/10] Update libraries/WebServer/examples/WebServer/data/index.htm Co-authored-by: Lucas Saavedra Vaz --- libraries/WebServer/examples/WebServer/data/index.htm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/WebServer/examples/WebServer/data/index.htm b/libraries/WebServer/examples/WebServer/data/index.htm index 58a04ede5be..06b48bf7038 100644 --- a/libraries/WebServer/examples/WebServer/data/index.htm +++ b/libraries/WebServer/examples/WebServer/data/index.htm @@ -22,4 +22,4 @@

Homepage of the WebServer Example

  • /api/list - Array of all files
    • - \ No newline at end of file + From 3f9badddb33297d856147800778e7dec393c5f8c Mon Sep 17 00:00:00 2001 From: Matthias Hertel Date: Thu, 30 Nov 2023 19:15:57 +0100 Subject: [PATCH 10/10] files.htm trailing newLine --- libraries/WebServer/examples/WebServer/data/files.htm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/WebServer/examples/WebServer/data/files.htm b/libraries/WebServer/examples/WebServer/data/files.htm index 4e8804154ac..95a8d8e3621 100644 --- a/libraries/WebServer/examples/WebServer/data/files.htm +++ b/libraries/WebServer/examples/WebServer/data/files.htm @@ -62,4 +62,4 @@

    Files on Server

    - \ No newline at end of file +