diff --git a/.github/workflows/LibraryBuild.yml b/.github/workflows/LibraryBuild.yml index a544ba7..2de3a93 100644 --- a/.github/workflows/LibraryBuild.yml +++ b/.github/workflows/LibraryBuild.yml @@ -34,7 +34,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index de92b20..72b1ccb 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: arduino/arduino-lint-action@v1 with: project-type: library diff --git a/.github/workflows/onrelease.yml b/.github/workflows/onrelease.yml index cf0c7cd..2d47ec2 100644 --- a/.github/workflows/onrelease.yml +++ b/.github/workflows/onrelease.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: ref: master # this must match the main/master branch name !! - name: Semver-Iterator diff --git a/README.md b/README.md index 10c0d04..5718d7b 100644 --- a/README.md +++ b/README.md @@ -445,6 +445,8 @@ Resources Alternate links --------------- + - [https://github.com/vortigont/esp32-flashz](https://github.com/vortigont/esp32-flashz) OTA-Update your ESP32 from zlib compressed binaries (not gzip) + - [https://github.com/chrisjoyce911/esp32FOTA](https://github.com/chrisjoyce911/esp32FOTA) OTA-Update your ESP32 from zlib or gzip compressed binaries - [https://github.com/laukik-hase/esp_compression](https://github.com/laukik-hase/esp_compression) inflate/deflate miniz/uzlib based esp-idf implementation Credits: @@ -458,5 +460,6 @@ Credits: - [lbernstone](https://github.com/lbernstone) (motivation and support) - [scubachristopher](https://github.com/scubachristopher) (contribution and support) - [infrafast](https://github.com/infrafast) (feedback fueler) + - [vortigont](https://github.com/vortigont/) (inspiration and support) diff --git a/examples/Test_tar_gz_tgz/Test_tar_gz_tgz.ino b/examples/Test_tar_gz_tgz/Test_tar_gz_tgz.ino index 83b6b5d..dd002b5 100644 --- a/examples/Test_tar_gz_tgz/Test_tar_gz_tgz.ino +++ b/examples/Test_tar_gz_tgz/Test_tar_gz_tgz.ino @@ -457,8 +457,11 @@ void setup() { Serial.begin( 115200 ); EEPROM.begin(512); + + #if defined DEST_FS_USES_SD - //SD.begin( 4 ); + SD.begin( 4 ); + //SD.begin(); #endif #ifdef ESP8266 diff --git a/library.properties b/library.properties index da4db85..a11299d 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=ESP32-targz -version=1.1.5 +version=1.1.6 author=tobozo maintainer=tobozo sentence=A library to unpack/uncompress tar, gz, and tar.gz files on ESP32 and ESP8266 diff --git a/src/ESP32-targz-lib.cpp b/src/ESP32-targz-lib.cpp index 9585c7d..85662b4 100644 --- a/src/ESP32-targz-lib.cpp +++ b/src/ESP32-targz-lib.cpp @@ -78,6 +78,7 @@ static void (*gzMessageCallback)( const char* format, ...) = nullptr; static void (*tarStatusProgressCallback)( const char* name, size_t size, size_t total_unpacked ) = nullptr; static void (*gzProgressCallback)( uint8_t progress ) = nullptr; static bool (*gzWriteCallback)( unsigned char* buff, size_t buffsize ) = nullptr; +static unsigned int (*gzReadDestByte)(int offset, unsigned char *out); static bool (*tarSkipThisEntryOut)( TAR::header_translated_t *header ) = nullptr; static bool (*tarSkipThisEntryIn)( TAR::header_translated_t *header ) = nullptr; static bool tarSkipThisEntry = false; @@ -92,6 +93,7 @@ static tarGzErrorCode _error = ESP32_TARGZ_OK; static bool targz_halt_on_error = false; static bool firstblock = true; // for gzProcessTarBuffer static bool lastblock = false; // for gzProcessTarBuffer +static uint32_t targz_read_timeout = 10000; // ms, should be larger than stream timeout static size_t tarCurrentFileSize = 0; static size_t tarCurrentFileSizeProgress = 0; static size_t tarTotalSize = 0; @@ -112,6 +114,7 @@ size_t min_output_buffer_size = 512; #endif #if defined ESP8266 static bool unTarDoHealthChecks = false; // ESP8266 is unstable with health checks + void vTaskDelay(int ms) { delay(ms); } // ESP8266 has no OS #endif @@ -167,6 +170,11 @@ BaseUnpacker::BaseUnpacker() } +void BaseUnpacker::setReadTimeout( uint32_t read_timeout ) +{ + targz_read_timeout = read_timeout; +} + #ifdef ESP32 bool BaseUnpacker::setPsram( bool enable ) { @@ -1023,7 +1031,10 @@ void GzUnpacker::setStreamWriter( gzStreamWriter cb ) gzWriteCallback = cb; } - +void GzUnpacker::setDestByteReader( gzDestByteReader cb ) +{ + gzReadDestByte = cb; +} void GzUnpacker::gzExpanderCleanup() { @@ -1116,7 +1127,7 @@ bool GzUnpacker::gzReadHeader( fs::File &gzFile ) // read a byte from the decompressed destination file, at 'offset' from the current position. // offset will be the negative offset back into the written output stream. // note: this does not ever write to the output stream; it simply reads from it. -unsigned int GzUnpacker::gzReadDestByte(int offset, unsigned char *out) +unsigned int GzUnpacker::gzReadDestByteFS(int offset, unsigned char *out) { unsigned char data; //delta between our position in output_buffer, and the desired offset in the output stream @@ -1143,13 +1154,19 @@ unsigned int GzUnpacker::gzReadDestByte(int offset, unsigned char *out) // returns 0 on success, or -1 on error. unsigned int GzUnpacker::gzReadSourceByte(CC_UNUSED struct GZ::TINF_DATA *data, unsigned char *out) { - //if( !BaseUnpacker::tarGzIO.gz->available() ) return -1; + _start: // using goto to avoid repeated code blocks if (tarGzIO.gz->readBytes( out, 1 ) != 1) { - log_v("readSourceByte read error, available is %d. attempting one-time retry", tarGzIO.gz->available()); - if (tarGzIO.gz->readBytes( out, 1 ) != 1) { - log_e("readSourceByte read error, available is %d. failed at retry", tarGzIO.gz->available()); - return -1; + uint32_t now = millis(); + uint32_t timeout = now + targz_read_timeout; + while( !tarGzIO.gz->available() ) { + if( millis()>timeout ) { + log_e("gz stream still unresponsive after %dms timeout, giving up", targz_read_timeout); + return -1; + } + vTaskDelay(1); // let the app breathe } + log_d("gz stream was unresponsive during %dms (timeout=%dms)", millis()-now, targz_read_timeout); + goto _start; } else { //log_v("read 1 byte: 0x%02x", out[0] ); } @@ -1214,7 +1231,7 @@ int GzUnpacker::gzUncompress( bool isupdate, bool stream_to_tar, bool use_dict, log_e("[ERROR] gz->tar->filesystem streaming requires a gzip dictionnnary"); return ESP32_TARGZ_NEEDS_DICT; } else { - uzLibDecompressor.readDestByte = gzReadDestByte; + uzLibDecompressor.readDestByte = gzReadDestByte ? gzReadDestByte : gzReadDestByteFS; log_v("[INFO] gz output is file"); } //output_buffer_size = SPI_FLASH_SEC_SIZE; @@ -1468,6 +1485,7 @@ bool GzUnpacker::gzStreamExpander( Stream *stream, size_t gz_size ) // TODO: ESP8266 support #if defined ESP8266 log_e("gz stream expanding not implemented on ESP8266"); + return false; #endif #if defined ESP32 @@ -2140,95 +2158,94 @@ bool TarGzUnpacker::tarGzStreamExpander( Stream *stream, fs::FS &destFS, const c #if defined ESP32 + /** GzUpdateClass Class implementation **/ -/** GzUpdateClass Class implementation **/ + bool GzUpdateClass::begingz(size_t size, int command, int ledPin, uint8_t ledOn, const char *label) + { + if( !gzProgressCallback ) { + log_d("Setting progress cb"); + gzUnpacker.setGzProgressCallback( gzUnpacker.defaultProgressCallback ); + } + if( !tgzLogger ) { + log_d("Setting logger cb"); + gzUnpacker.setLoggerCallback( gzUnpacker.targzPrintLoggerCallback ); + } + if( gzWriteCallback == nullptr ) { + log_d("Setting write cb"); + gzUnpacker.setStreamWriter( this->gzUpdateWriteCallback ); + } -bool GzUpdateClass::begingz(size_t size, int command, int ledPin, uint8_t ledOn, const char *label) -{ - if( !gzProgressCallback ) { - log_d("Setting progress cb"); - gzUnpacker.setGzProgressCallback( gzUnpacker.defaultProgressCallback ); - } - if( !tgzLogger ) { - log_d("Setting logger cb"); - gzUnpacker.setLoggerCallback( gzUnpacker.targzPrintLoggerCallback ); - } - if( gzWriteCallback == nullptr ) { - log_d("Setting write cb"); - gzUnpacker.setStreamWriter( gzUpdateWriteCallback ); - //gzUnpacker.setStreamWriter( Update.write ); - } + mode_gz = true; - mode_gz = true; + bool ret = begin(size, command, ledPin, ledOn, label); - bool ret = begin(size, command, ledPin, ledOn, label); + return ret; + } - return ret; -} + bool GzUpdateClass::gzUpdateWriteCallback( unsigned char* buff, size_t buffsize ) + { + int written = GzUpdateClass::getInstance().write( buff, buffsize ); + if( written ) { + log_v("Wrote %d bytes", written ); + return true; + } + log_e("Failed to write %d bytes", buffsize ); + return false; + } -bool GzUpdateClass::gzUpdateWriteCallback( unsigned char* buff, size_t buffsize ) -{ - if( GzUpdateClass::getInstance().write( buff, buffsize ) ) { - log_v("Wrote %d bytes", buffsize ); - return true; + + void GzUpdateClass::abortgz() + { + abort(); + gzUnpacker.gzExpanderCleanup(); + mode_gz = false; } - log_e("Failed to write %d bytes", buffsize ); - return false; -} -void GzUpdateClass::abortgz() -{ - abort(); - gzUnpacker.gzExpanderCleanup(); - mode_gz = false; -} + bool GzUpdateClass::endgz(bool evenIfRemaining) + { + gzUnpacker.gzExpanderCleanup(); + mode_gz = false; + return end(evenIfRemaining); + } -bool GzUpdateClass::endgz(bool evenIfRemaining) -{ - gzUnpacker.gzExpanderCleanup(); - mode_gz = false; - return end(evenIfRemaining); -} + size_t GzUpdateClass::writeGzStream(Stream &data, size_t len) + { + if (!mode_gz) { + log_d("Not in gz mode"); + return writeStream(data); + } + uint32_t timeout = millis() + targz_read_timeout; -size_t GzUpdateClass::writeGzStream(Stream &data, size_t len) -{ - if (!mode_gz) { - log_d("Not in gz mode"); - return writeStream(data); - } + while( !data.available() ) { + if(millis()>timeout) { + log_e("stream still not responsive after %dms timeout, giving up", targz_read_timeout); + return 0; + } + vTaskDelay(1); + } - size_t size = data.available(); - if( ! size ) { - log_e("Bad stream, aborting"); - //gzUnpacker.setError( ESP32_TARGZ_STREAM_ERROR ); - return 0; - } + log_d("In gz mode"); - log_d("In gz mode"); + tarGzIO.gz = &data; - tarGzIO.gz = &data; - // process with unzipping - bool show_progress = false; - bool use_dict = true; - bool isupdate = true; - bool stream_to_tar = false; - int ret = gzUnpacker.gzUncompress( isupdate, stream_to_tar, use_dict, show_progress ); - // unzipping ended - if( ret!=0 ) { - log_e("gzHTTPUpdater returned error code %d", ret); - //gzUnpacker.setError( (tarGzErrorCode)ret ); - return 0; - } + // process with decompressing + int ret = gzUnpacker.gzUncompress( true/*isupdate*/, false/*stream_to_tar*/, true/*use_dict*/, false/*show_progress*/ ); + + if( ret!=0 ) { + log_e("gzUncompress returned error code %d (free heap=%d bytes)", ret, ESP.getFreeHeap() ); + //gzUnpacker.setError( (tarGzErrorCode)ret ); + return 0; + } - log_d("unpack complete (%d bytes)", tarGzIO.gz_size ); + //log_d("unpack complete (%d bytes)", use_dict ? tarGzIO.gz_size : GzUpdateClass_Write_Offset ); - return len; -} + return len; + } #endif diff --git a/src/ESP32-targz-lib.hpp b/src/ESP32-targz-lib.hpp index 0e8992e..7739d09 100644 --- a/src/ESP32-targz-lib.hpp +++ b/src/ESP32-targz-lib.hpp @@ -100,6 +100,8 @@ typedef void (*fsSetupCallbacks)( fsTotalBytesCb cbt, fsFreeBytesCb cbf ); // overridable gz stream writer typedef bool (*gzStreamWriter)( unsigned char* buff, size_t buffsize ); +// overridable gz byte reader (used when no dictionary set) +typedef unsigned int (*gzDestByteReader)(int offset, unsigned char *out); // tar doesn't have a real progress, so provide a status instead typedef void (*tarStatusProgressCb)( const char* name, size_t size, size_t total_unpacked ); @@ -205,6 +207,7 @@ struct BaseUnpacker static void setFsTotalBytesCb( fsTotalBytesCb cb ); // filesystem helpers totalBytes static void setFsFreeBytesCb( fsFreeBytesCb cb ); // filesystem helpers freeBytes static void setGeneralError( tarGzErrorCode code ); // alias to static setError + static void setReadTimeout( uint32_t read_timeout ); // read timeout: set high value (e.g. 10000ms) network and low value for filesystems }; @@ -249,6 +252,7 @@ struct GzUnpacker : virtual public BaseUnpacker void setGzMessageCallback( genericLoggerCallback cb ); //void setStreamReader( gzStreamReader cb ); // optional, use with gzStreamExpander void setStreamWriter( gzStreamWriter cb ); // optional, use with gzStreamExpander + void setDestByteReader( gzDestByteReader cb ); void gzExpanderCleanup(); int gzUncompress( bool isupdate = false, bool stream_to_tar = false, bool use_dict = true, bool show_progress = true ); @@ -256,7 +260,7 @@ struct GzUnpacker : virtual public BaseUnpacker static bool gzStreamWriteCallback( unsigned char* buff, size_t buffsize ); static bool gzReadHeader(fs::File &gzFile); static uint8_t gzReadByte(fs::File &gzFile, const int32_t addr, fs::SeekMode mode=fs::SeekSet); - static unsigned int gzReadDestByte(int offset, unsigned char *out); + static unsigned int gzReadDestByteFS(int offset, unsigned char *out); static unsigned int gzReadSourceByte(struct GZ::TINF_DATA *data, unsigned char *out); }; @@ -285,79 +289,81 @@ struct TarGzUnpacker : public TarUnpacker, public GzUnpacker #if defined ESP32 -class GzUpdateClass : public UpdateClass { - - GzUpdateClass(){}; // hidden c-tor - ~GzUpdateClass(){}; // hidden d-tor - - bool mode_gz = false; // needed to keep mode state for async writez() calls - int command = U_FLASH; // needed to keep track of the destination partition - GzUnpacker gzUnpacker; - - /** - * @brief callback for GzUnpacker - * writes inflated firmware chunk to flash - * - */ - //int flash_cb(size_t index, const uint8_t* data, size_t size, bool final); //> inflate_cb_t - - public: - // this is a singleton, no copy's - GzUpdateClass(const GzUpdateClass&) = delete; - GzUpdateClass& operator=(const GzUpdateClass &) = delete; - GzUpdateClass(GzUpdateClass &&) = delete; - GzUpdateClass & operator=(GzUpdateClass &&) = delete; - - static GzUpdateClass& getInstance(){ - static GzUpdateClass flashz; - return flashz; - } - - /** - * @brief initilize GzUnpacker structs and UpdaterClass - * - * @return true on success - * @return false on GzUnpacker mem allocation error or flash free space error - */ - bool begingz(size_t size=UPDATE_SIZE_UNKNOWN, int command = U_FLASH, int ledPin = -1, uint8_t ledOn = LOW, const char *label = NULL); - - /** - * @brief Writes a buffer to the flash - * Returns true on success - * - * @param buff - * @param buffsize - * @return processed bytes - */ - //size_t writez(const uint8_t *data, size_t len, bool final); - static bool gzUpdateWriteCallback( unsigned char* buff, size_t buffsize ); - - /** - * @brief Read zlib compressed data from stream, decompress and write it to flash - * size of the stream must be known in order to signal zlib inflator last chunk - * - * @param data Stream object, usually data from a tcp socket - * @param len total length of compressed data to read from stream - * @return size_t number of bytes processed from a stream - */ - size_t writeGzStream(Stream &data, size_t len); - - /** - * @brief abort running inflator and flash update process - * also releases inflator memory - */ - void abortgz(); - - /** - * @brief release inflator memory and run UpdateClass.end() - * returns status of end() call - * - * @return true - * @return false - */ - bool endgz(bool evenIfRemaining = true); - -}; + // this class was inspired by https://github.com/vortigont/esp32-flashz + + class GzUpdateClass : public UpdateClass { + + GzUpdateClass(){}; // hidden c-tor + ~GzUpdateClass(){}; // hidden d-tor + + bool mode_gz = false; // needed to keep mode state for async writez() calls + int command = U_FLASH; // needed to keep track of the destination partition + GzUnpacker gzUnpacker; + + /** + * @brief callback for GzUnpacker + * writes inflated firmware chunk to flash + * + */ + //int flash_cb(size_t index, const uint8_t* data, size_t size, bool final); //> inflate_cb_t + + public: + // this is a singleton, no copy's + GzUpdateClass(const GzUpdateClass&) = delete; + GzUpdateClass& operator=(const GzUpdateClass &) = delete; + GzUpdateClass(GzUpdateClass &&) = delete; + GzUpdateClass & operator=(GzUpdateClass &&) = delete; + + static GzUpdateClass& getInstance(){ + static GzUpdateClass flashz; + return flashz; + } + + /** + * @brief initilize GzUnpacker structs and UpdaterClass + * + * @return true on success + * @return false on GzUnpacker mem allocation error or flash free space error + */ + bool begingz(size_t size=UPDATE_SIZE_UNKNOWN, int command = U_FLASH, int ledPin = -1, uint8_t ledOn = LOW, const char *label = NULL); + + /** + * @brief Writes a buffer to the flash + * Returns true on success + * + * @param buff + * @param buffsize + * @return processed bytes + */ + //size_t writez(const uint8_t *data, size_t len, bool final); + static bool gzUpdateWriteCallback( unsigned char* buff, size_t buffsize ); + + /** + * @brief Read zlib compressed data from stream, decompress and write it to flash + * size of the stream must be known in order to signal zlib inflator last chunk + * + * @param data Stream object, usually data from a tcp socket + * @param len total length of compressed data to read from stream + * @return size_t number of bytes processed from a stream + */ + size_t writeGzStream(Stream &data, size_t len); + + /** + * @brief abort running inflator and flash update process + * also releases inflator memory + */ + void abortgz(); + + /** + * @brief release inflator memory and run UpdateClass.end() + * returns status of end() call + * + * @return true + * @return false + */ + bool endgz(bool evenIfRemaining = true); + + }; #endif diff --git a/src/ESP32-targz.h b/src/ESP32-targz.h index b2570f7..0cf3cea 100644 --- a/src/ESP32-targz.h +++ b/src/ESP32-targz.h @@ -38,12 +38,7 @@ #define tarGzFS PSRamFS #define FS_NAME "PSRamFS" #else - #warning "Unspecified filesystem, please #define one of these before including the library: DEST_FS_USES_SPIFFS, DEST_FS_USES_FFAT, DEST_FS_USES_SD, DEST_FS_USES_SD_MMC, DEST_FS_USES_LITTLEFS" - #warning "Defaulting to SPIFFS" - #define DEST_FS_USES_SPIFFS - #include - #define tarGzFS SPIFFS - #define FS_NAME "SPIFFS" + // no filesystem, no helpers available, power user ? #endif #elif defined ESP8266 @@ -56,7 +51,7 @@ #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #if defined DEST_FS_USES_SD - #include + #include #define tarGzFS SDFS #define FS_NAME "SDFS" #else @@ -125,7 +120,8 @@ __attribute__((unused)) static size_t targzFreeBytesFn() { #elif defined DEST_FS_USES_FFAT return tarGzFS.freeBytes(); #else - #error "No filesystem is declared" + // no filesystem, no helpers available, power user ? + return 0; #endif } @@ -144,7 +140,8 @@ __attribute__((unused)) static size_t targzTotalBytesFn() { #error "Only ESP32 and ESP8266 are supported" #endif #else - #error "No filesystem is declared" + // no filesystem, no helpers available, power user ? + return 0; #endif }