From de751931c13d76a20499a035fc1cb419a3a8ff6b Mon Sep 17 00:00:00 2001 From: Nathan Nau Date: Thu, 28 Mar 2024 10:21:45 +0100 Subject: [PATCH 1/5] Handle large octet-stream --- libraries/WebServer/src/Parsing.cpp | 28 ++++++++++++++++++- libraries/WebServer/src/WebServer.cpp | 1 + libraries/WebServer/src/WebServer.h | 16 +++++++++++ .../WebServer/src/detail/RequestHandler.h | 2 ++ .../src/detail/RequestHandlersImpl.h | 13 +++++++++ 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/libraries/WebServer/src/Parsing.cpp b/libraries/WebServer/src/Parsing.cpp index f3b19b19d6e..6833b7d9dc2 100644 --- a/libraries/WebServer/src/Parsing.cpp +++ b/libraries/WebServer/src/Parsing.cpp @@ -136,6 +136,7 @@ bool WebServer::_parseRequest(WiFiClient& client) { String headerName; String headerValue; bool isForm = false; + bool isRaw = false; bool isEncoded = false; //parse headers while(1){ @@ -158,6 +159,8 @@ bool WebServer::_parseRequest(WiFiClient& client) { using namespace mime; if (headerValue.startsWith(FPSTR(mimeTable[txt].mimeType))){ isForm = false; + } else if (headerValue.startsWith(FPSTR(mimeTable[none].mimeType))){ + isRaw = true; } else if (headerValue.startsWith(F("application/x-www-form-urlencoded"))){ isForm = false; isEncoded = true; @@ -173,7 +176,30 @@ bool WebServer::_parseRequest(WiFiClient& client) { } } - if (!isForm){ + if (isRaw && _currentHandler && _currentHandler->canRaw(_currentUri)){ + log_v("Parse raw"); + _currentRaw.reset(new HTTPRaw()); + _currentRaw->status = RAW_START; + _currentRaw->totalSize = 0; + _currentRaw->currentSize = 0; + log_v("Start Raw"); + _currentHandler->raw(*this, _currentUri, *_currentRaw); + _currentRaw->status = RAW_WRITE; + + while (_currentRaw->totalSize < _clientContentLength) { + _currentRaw->currentSize = client.readBytes(_currentRaw->buf, HTTP_RAW_BUFLEN); + _currentRaw->totalSize += _currentRaw->currentSize; + if (_currentRaw->currentSize == 0) { + _currentRaw->status = RAW_ABORTED; + _currentHandler->raw(*this, _currentUri, *_currentRaw); + return false; + } + _currentHandler->raw(*this, _currentUri, *_currentRaw); + } + _currentRaw->status = RAW_END; + _currentHandler->raw(*this, _currentUri, *_currentRaw); + log_v("Finish Raw"); + } else if (!isForm) { size_t plainLength; char* plainBuf = readBytesWithTimeout(client, _clientContentLength, plainLength, HTTP_MAX_POST_WAIT); if (plainLength < _clientContentLength) { diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index 8b3391d15c8..d296ce716b0 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -333,6 +333,7 @@ void WebServer::handleClient() { _currentClient = WiFiClient(); _currentStatus = HC_NONE; _currentUpload.reset(); + _currentRaw.reset(); } if (callYield) { diff --git a/libraries/WebServer/src/WebServer.h b/libraries/WebServer/src/WebServer.h index fc60d16496f..d472d8d772e 100644 --- a/libraries/WebServer/src/WebServer.h +++ b/libraries/WebServer/src/WebServer.h @@ -32,6 +32,7 @@ enum HTTPUploadStatus { UPLOAD_FILE_START, UPLOAD_FILE_WRITE, UPLOAD_FILE_END, UPLOAD_FILE_ABORTED }; +enum HTTPRawStatus { RAW_START, RAW_WRITE, RAW_END, RAW_ABORTED }; enum HTTPClientStatus { HC_NONE, HC_WAIT_READ, HC_WAIT_CLOSE }; enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH }; @@ -41,6 +42,10 @@ enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH }; #define HTTP_UPLOAD_BUFLEN 1436 #endif +#ifndef HTTP_RAW_BUFLEN +#define HTTP_RAW_BUFLEN 1436 +#endif + #define HTTP_MAX_DATA_WAIT 5000 //ms to wait for the client to send the request #define HTTP_MAX_POST_WAIT 5000 //ms to wait for POST data to arrive #define HTTP_MAX_SEND_WAIT 5000 //ms to wait for data chunk to be ACKed @@ -61,6 +66,15 @@ typedef struct { uint8_t buf[HTTP_UPLOAD_BUFLEN]; } HTTPUpload; +typedef struct +{ + HTTPRawStatus status; + size_t totalSize; // content size + size_t currentSize; // size of data currently in buf + uint8_t buf[HTTP_UPLOAD_BUFLEN]; + void *data; // additional data +} HTTPRaw; + #include "detail/RequestHandler.h" namespace fs { @@ -97,6 +111,7 @@ class WebServer HTTPMethod method() { return _currentMethod; } virtual WiFiClient client() { return _currentClient; } HTTPUpload& upload() { return *_currentUpload; } + HTTPRaw& raw() { return *_currentRaw; } String pathArg(unsigned int i); // get request path argument by number String arg(String name); // get request argument value by name @@ -196,6 +211,7 @@ class WebServer RequestArgument* _postArgs; std::unique_ptr _currentUpload; + std::unique_ptr _currentRaw; int _headerKeysCount; RequestArgument* _currentHeaders; diff --git a/libraries/WebServer/src/detail/RequestHandler.h b/libraries/WebServer/src/detail/RequestHandler.h index 871ae5c8b3d..27ca3c9c771 100644 --- a/libraries/WebServer/src/detail/RequestHandler.h +++ b/libraries/WebServer/src/detail/RequestHandler.h @@ -9,8 +9,10 @@ class RequestHandler { virtual ~RequestHandler() { } virtual bool canHandle(HTTPMethod method, String uri) { (void) method; (void) uri; return false; } virtual bool canUpload(String uri) { (void) uri; return false; } + virtual bool canRaw(String uri) { (void) uri; return false; } virtual bool handle(WebServer& server, HTTPMethod requestMethod, String requestUri) { (void) server; (void) requestMethod; (void) requestUri; return false; } virtual void upload(WebServer& server, String requestUri, HTTPUpload& upload) { (void) server; (void) requestUri; (void) upload; } + virtual void raw(WebServer& server, String requestUri, HTTPRaw& raw) { (void) server; (void) requestUri; (void) raw; } RequestHandler* next() { return _next; } void next(RequestHandler* r) { _next = r; } diff --git a/libraries/WebServer/src/detail/RequestHandlersImpl.h b/libraries/WebServer/src/detail/RequestHandlersImpl.h index 4a7c28e58ae..d3c9be10996 100644 --- a/libraries/WebServer/src/detail/RequestHandlersImpl.h +++ b/libraries/WebServer/src/detail/RequestHandlersImpl.h @@ -36,6 +36,12 @@ class FunctionRequestHandler : public RequestHandler { return true; } + bool canRaw(String requestUri) override { + if (!_ufn || _method == HTTP_GET) + return false; + + return true; + } bool handle(WebServer& server, HTTPMethod requestMethod, String requestUri) override { (void) server; @@ -53,6 +59,13 @@ class FunctionRequestHandler : public RequestHandler { _ufn(); } + void raw(WebServer& server, String requestUri, HTTPRaw& raw) override { + (void)server; + (void)raw; + if (canRaw(requestUri)) + _ufn(); + } + protected: WebServer::THandlerFunction _fn; WebServer::THandlerFunction _ufn; From 2dd14e993cd15ededd5027476a7450d223304d43 Mon Sep 17 00:00:00 2001 From: Nathan Nau Date: Tue, 2 Apr 2024 09:42:14 +0200 Subject: [PATCH 2/5] Add exemple Upload Huge File --- .../examples/UploadHugeFile/README.md | 13 ++ .../UploadHugeFile/UploadHugeFile.ino | 132 ++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 libraries/WebServer/examples/UploadHugeFile/README.md create mode 100644 libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino diff --git a/libraries/WebServer/examples/UploadHugeFile/README.md b/libraries/WebServer/examples/UploadHugeFile/README.md new file mode 100644 index 00000000000..08dad0e4222 --- /dev/null +++ b/libraries/WebServer/examples/UploadHugeFile/README.md @@ -0,0 +1,13 @@ +# Upload Huge File To SD Over Http + +This project is an example of an HTTP server designed to facilitate the transfer of large files using the PUT method, in accordance with RFC specifications. + +### Example cURL Command + +```bash +curl -X PUT -H "Content-Type: application/octet-stream" -T ./my-file.mp3 http://esp-ip/upload/my-file.mp3 +``` + +## Resources + +- RFC HTTP/1.0 - Additional Request Methods - PUT : [Link](https://datatracker.ietf.org/doc/html/rfc1945#appendix-D.1.1) diff --git a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino new file mode 100644 index 00000000000..091bbfab631 --- /dev/null +++ b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino @@ -0,0 +1,132 @@ +#include +#include +#include +#include +#include + +const char* ssid = "**********"; +const char* password = "**********"; + +WebServer server(80); + +File rawFile; +void handleCreate() { + server.send(200, "text/plain", ""); +} +void handleCreateProcess() { + String path = server.pathArg(0); + HTTPRaw& raw = server.raw(); + if (raw.status == RAW_START) { + if (SD.exists((char *)path.c_str())) { + SD.remove((char *)path.c_str()); + } + rawFile = SD.open(path.c_str(), FILE_WRITE); + Serial.print("Upload: START, filename: "); + Serial.println(path); + } else if (raw.status == RAW_WRITE) { + if (rawFile) { + rawFile.write(raw.buf, raw.currentSize); + } + Serial.print("Upload: WRITE, Bytes: "); + Serial.println(raw.currentSize); + } else if (raw.status == RAW_END) { + if (rawFile) { + rawFile.close(); + } + Serial.print("Upload: END, Size: "); + Serial.println(raw.totalSize); + } +} + +void returnFail(String msg) { + server.send(500, "text/plain", msg + "\r\n"); +} + +void printDirectory() { + if (!server.hasArg("dir")) { + return returnFail("BAD ARGS"); + } + String path = server.arg("dir"); + if (path != "/" && !SD.exists((char *)path.c_str())) { + return returnFail("BAD PATH"); + } + File dir = SD.open((char *)path.c_str()); + path = String(); + if (!dir.isDirectory()) { + dir.close(); + return returnFail("NOT DIR"); + } + dir.rewindDirectory(); + server.setContentLength(CONTENT_LENGTH_UNKNOWN); + server.send(200, "text/json", ""); + WiFiClient client = server.client(); + + server.sendContent("["); + for (int cnt = 0; true; ++cnt) { + File entry = dir.openNextFile(); + if (!entry) { + break; + } + + String output; + if (cnt > 0) { + output = ','; + } + + output += "{\"type\":\""; + output += (entry.isDirectory()) ? "dir" : "file"; + output += "\",\"name\":\""; + output += entry.path(); + output += "\""; + output += "}"; + server.sendContent(output); + entry.close(); + } + server.sendContent("]"); + dir.close(); +} + +void handleNotFound() { + String message = "File Not Found\n\n"; + message += "URI: "; + message += server.uri(); + message += "\nMethod: "; + message += (server.method() == HTTP_GET) ? "GET" : "POST"; + message += "\nArguments: "; + message += server.args(); + message += "\n"; + for (uint8_t i = 0; i < server.args(); i++) { + message += " " + server.argName(i) + ": " + server.arg(i) + "\n"; + } + server.send(404, "text/plain", message); +} + +void setup(void) { + Serial.begin(115200); + + while (!SD.begin()) delay(1); + Serial.println("SD Card initialized."); + + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + + while (WiFi.status() != WL_CONNECTED) { + delay(500); + Serial.print("."); + } + Serial.print("Connected to "); + Serial.println(ssid); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); + + server.on(UriRegex("/upload/(.*)"), HTTP_PUT, handleCreate, handleCreateProcess); + server.onNotFound(handleNotFound); + server.begin(); + Serial.println("HTTP server started"); + +} + +void loop(void) { + server.handleClient(); + delay(2);//allow the cpu to switch to other tasks +} From 40c912644a60f2a25293bb4668144c91c1aa3120 Mon Sep 17 00:00:00 2001 From: Nathan Nau Date: Sat, 6 Apr 2024 19:14:16 +0200 Subject: [PATCH 3/5] Remove unuse function printDirectory --- .../UploadHugeFile/UploadHugeFile.ino | 44 ------------------- 1 file changed, 44 deletions(-) diff --git a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino index 091bbfab631..58014706908 100644 --- a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino +++ b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino @@ -42,50 +42,6 @@ void returnFail(String msg) { server.send(500, "text/plain", msg + "\r\n"); } -void printDirectory() { - if (!server.hasArg("dir")) { - return returnFail("BAD ARGS"); - } - String path = server.arg("dir"); - if (path != "/" && !SD.exists((char *)path.c_str())) { - return returnFail("BAD PATH"); - } - File dir = SD.open((char *)path.c_str()); - path = String(); - if (!dir.isDirectory()) { - dir.close(); - return returnFail("NOT DIR"); - } - dir.rewindDirectory(); - server.setContentLength(CONTENT_LENGTH_UNKNOWN); - server.send(200, "text/json", ""); - WiFiClient client = server.client(); - - server.sendContent("["); - for (int cnt = 0; true; ++cnt) { - File entry = dir.openNextFile(); - if (!entry) { - break; - } - - String output; - if (cnt > 0) { - output = ','; - } - - output += "{\"type\":\""; - output += (entry.isDirectory()) ? "dir" : "file"; - output += "\",\"name\":\""; - output += entry.path(); - output += "\""; - output += "}"; - server.sendContent(output); - entry.close(); - } - server.sendContent("]"); - dir.close(); -} - void handleNotFound() { String message = "File Not Found\n\n"; message += "URI: "; From 020042e20548b53441aa2611837dd789f712a2be Mon Sep 17 00:00:00 2001 From: Nathan Nau Date: Sat, 6 Apr 2024 19:15:37 +0200 Subject: [PATCH 4/5] Fix upload path --- libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino index 58014706908..c05b234af51 100644 --- a/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino +++ b/libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino @@ -14,7 +14,7 @@ void handleCreate() { server.send(200, "text/plain", ""); } void handleCreateProcess() { - String path = server.pathArg(0); + String path = "/" + server.pathArg(0); HTTPRaw& raw = server.raw(); if (raw.status == RAW_START) { if (SD.exists((char *)path.c_str())) { From 30c309cffa352fa9dd53821f92946d990f592ca1 Mon Sep 17 00:00:00 2001 From: Nathan Nau Date: Sat, 6 Apr 2024 19:30:25 +0200 Subject: [PATCH 5/5] Simplify and generalize the body parsing. --- libraries/WebServer/examples/UploadHugeFile/README.md | 2 +- libraries/WebServer/src/Parsing.cpp | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/libraries/WebServer/examples/UploadHugeFile/README.md b/libraries/WebServer/examples/UploadHugeFile/README.md index 08dad0e4222..607dc9d71d8 100644 --- a/libraries/WebServer/examples/UploadHugeFile/README.md +++ b/libraries/WebServer/examples/UploadHugeFile/README.md @@ -5,7 +5,7 @@ This project is an example of an HTTP server designed to facilitate the transfer ### Example cURL Command ```bash -curl -X PUT -H "Content-Type: application/octet-stream" -T ./my-file.mp3 http://esp-ip/upload/my-file.mp3 +curl -X PUT -T ./my-file.mp3 http://esp-ip/upload/my-file.mp3 ``` ## Resources diff --git a/libraries/WebServer/src/Parsing.cpp b/libraries/WebServer/src/Parsing.cpp index 6833b7d9dc2..f0b6692bc0c 100644 --- a/libraries/WebServer/src/Parsing.cpp +++ b/libraries/WebServer/src/Parsing.cpp @@ -136,7 +136,6 @@ bool WebServer::_parseRequest(WiFiClient& client) { String headerName; String headerValue; bool isForm = false; - bool isRaw = false; bool isEncoded = false; //parse headers while(1){ @@ -159,8 +158,6 @@ bool WebServer::_parseRequest(WiFiClient& client) { using namespace mime; if (headerValue.startsWith(FPSTR(mimeTable[txt].mimeType))){ isForm = false; - } else if (headerValue.startsWith(FPSTR(mimeTable[none].mimeType))){ - isRaw = true; } else if (headerValue.startsWith(F("application/x-www-form-urlencoded"))){ isForm = false; isEncoded = true; @@ -176,7 +173,7 @@ bool WebServer::_parseRequest(WiFiClient& client) { } } - if (isRaw && _currentHandler && _currentHandler->canRaw(_currentUri)){ + if (!isForm && _currentHandler && _currentHandler->canRaw(_currentUri)){ log_v("Parse raw"); _currentRaw.reset(new HTTPRaw()); _currentRaw->status = RAW_START;