Merge pull request #166 from arduino-libraries/lzss-tooling

Add tool for LZSS compression/decompression and improve documentation

extras/tools/.gitignore

@@ -0,0 +1,4 @@
+*.bin
+*.ota
+*.json
+

extras/tools/README.md

@@ -1,5 +1,18 @@
-`bin2ota.py`
-============
+Firmware Over-The-Air Tools
+===========================
+
+## How-to-OTA
+
+Arduino IDE: `Sketch` -> `Export compiled Binary`
+```bash
+cp sketch.bin ~/Arduino/libraries/ArduinoIoTCloud/extras/tools/
+cd ~/Arduino/libraries/ArduinoIoTCloud/extras/tools
+./bin2ota.py sketch.bin sketch.ota
+./bin2json.py sketch.ota sketch.json
+./ota-upload.sh CLIENT_ID CLIENT_SECRET DEVICE_ID sketch.json
+```
+
+## `bin2ota.py`
 This tool can be used to extend (actually prefix) a binary generated with e.g. the Arduino IDE with the required length and crc values required to perform an OTA (Over-The-Air) update of the firmware.
 
 ### How-To-Use
@@ -25,17 +38,28 @@ This tool can be used to extend (actually prefix) a binary generated with e.g. t
 0000030   0000 0000 7485 0000 0000 0000 0000 0000
 ```
 
-`bin2json.py`
-=============
+## `lzss.py`
+This tool allows to compress a binary file using the LZSS algorithm.
+
+### How-To-Use
+* Encoding (Compressing)
+```bash
+./lzss.py --encode sketch.bin sketch.lzss
+```
+* Decoding (Extracting)
+```bash
+./lzss.py --decode sketch.lzss sketch.bin
+```
+
+## `bin2json.py`
 This tool converts the binary file into base64 encoded JSON which is necessary for feeding it to the server.
 
 ### How-To-Use
 ```bash
 ./bin2json.py sketch.ota sketch.json
 ```
 
-`ota-upload.sh`
-==============
+## `ota-upload.sh`
 This tool allows to upload a OTA binary to a device via a Arduino cloud server.
 
 ### How-To-Use

extras/tools/lzss.c

@@ -0,0 +1,186 @@
+/* LZSS encoder-decoder (Haruhiko Okumura; public domain) */
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#define EI 11  /* typically 10..13 */
+#define EJ  4  /* typically 4..5 */
+#define P   1  /* If match length <= P then output one character */
+#define N (1 << EI)  /* buffer size */
+#define F ((1 << EJ) + 1)  /* lookahead buffer size */
+
+int bit_buffer = 0, bit_mask = 128;
+unsigned long codecount = 0, textcount = 0;
+unsigned char buffer[N * 2];
+FILE *infile, *outfile;
+
+void error(void)
+{
+    printf("Output error\n");  exit(1);
+}
+
+void putbit1(void)
+{
+    bit_buffer |= bit_mask;
+    if ((bit_mask >>= 1) == 0) {
+        if (fputc(bit_buffer, outfile) == EOF) error();
+        bit_buffer = 0;  bit_mask = 128;  codecount++;
+    }
+}
+
+void putbit0(void)
+{
+    if ((bit_mask >>= 1) == 0) {
+        if (fputc(bit_buffer, outfile) == EOF) error();
+        bit_buffer = 0;  bit_mask = 128;  codecount++;
+    }
+}
+
+void flush_bit_buffer(void)
+{
+    if (bit_mask != 128) {
+        if (fputc(bit_buffer, outfile) == EOF) error();
+        codecount++;
+    }
+}
+
+void output1(int c)
+{
+    int mask;
+
+    putbit1();
+    mask = 256;
+    while (mask >>= 1) {
+        if (c & mask) putbit1();
+        else putbit0();
+    }
+}
+
+void output2(int x, int y)
+{
+    int mask;
+
+    putbit0();
+    mask = N;
+    while (mask >>= 1) {
+        if (x & mask) putbit1();
+        else putbit0();
+    }
+    mask = (1 << EJ);
+    while (mask >>= 1) {
+        if (y & mask) putbit1();
+        else putbit0();
+    }
+}
+
+void encode(void)
+{
+    int i, j, f1, x, y, r, s, bufferend, c;
+
+    for (i = 0; i < N - F; i++) buffer[i] = ' ';
+    for (i = N - F; i < N * 2; i++) {
+        if ((c = fgetc(infile)) == EOF) break;
+        buffer[i] = c;  textcount++;
+    }
+    bufferend = i;  r = N - F;  s = 0;
+    while (r < bufferend) {
+        f1 = (F <= bufferend - r) ? F : bufferend - r;
+        x = 0;  y = 1;  c = buffer[r];
+        for (i = r - 1; i >= s; i--)
+            if (buffer[i] == c) {
+                for (j = 1; j < f1; j++)
+                    if (buffer[i + j] != buffer[r + j]) break;
+                if (j > y) {
+                    x = i;  y = j;
+                }
+            }
+        if (y <= P) {  y = 1;  output1(c);  }
+        else output2(x & (N - 1), y - 2);
+        r += y;  s += y;
+        if (r >= N * 2 - F) {
+            for (i = 0; i < N; i++) buffer[i] = buffer[i + N];
+            bufferend -= N;  r -= N;  s -= N;
+            while (bufferend < N * 2) {
+                if ((c = fgetc(infile)) == EOF) break;
+                buffer[bufferend++] = c;  textcount++;
+            }
+        }
+    }
+    flush_bit_buffer();
+    printf("text:  %ld bytes\n", textcount);
+    printf("code:  %ld bytes (%ld%%)\n",
+        codecount, (codecount * 100) / textcount);
+}
+
+int getbit(int n) /* get n bits */
+{
+    int i, x;
+    static int buf, mask = 0;
+
+    x = 0;
+    for (i = 0; i < n; i++) {
+        if (mask == 0) {
+            if ((buf = fgetc(infile)) == EOF) return EOF;
+            mask = 128;
+        }
+        x <<= 1;
+        if (buf & mask) x++;
+        mask >>= 1;
+    }
+    return x;
+}
+
+void decode(void)
+{
+    int i, j, k, r, c;
+
+    for (i = 0; i < N - F; i++) buffer[i] = ' ';
+    r = N - F;
+    while ((c = getbit(1)) != EOF) {
+        if (c) {
+            if ((c = getbit(8)) == EOF) break;
+            fputc(c, outfile);
+            buffer[r++] = c;  r &= (N - 1);
+        } else {
+            if ((i = getbit(EI)) == EOF) break;
+            if ((j = getbit(EJ)) == EOF) break;
+            for (k = 0; k <= j + 1; k++) {
+                c = buffer[(i + k) & (N - 1)];
+                fputc(c, outfile);
+                buffer[r++] = c;  r &= (N - 1);
+            }
+        }
+    }
+}
+
+int encode_file(char const * in, char const * out)
+{
+    infile = fopen(in, "rb");
+    if (infile == NULL) return 0;
+
+    outfile = fopen(out, "wb");
+    if (outfile == NULL) return 0;
+
+    encode();
+
+    fclose(infile);
+    fclose(outfile);
+
+    return 0;
+}
+
+int decode_file(char const * in, char const * out)
+{
+    infile = fopen(in, "rb");
+    if (infile == NULL) return 0;
+
+    outfile = fopen(out, "wb");
+    if (outfile == NULL) return 0;
+
+    decode();
+
+    fclose(infile);
+    fclose(outfile);
+
+    return 0;
+}

extras/tools/lzss.py

@@ -0,0 +1,28 @@
+#!/usr/bin/python3
+
+import sys
+import ctypes
+
+LZSS_SO_FILE = "./lzss.so"
+
+if len(sys.argv) != 4:
+    print ("Usage: lzss.py --[encode|decode] infile outfile")
+    sys.exit()
+
+lzss_functions = ctypes.CDLL(LZSS_SO_FILE)
+
+mode   = sys.argv[1]
+ifile  = sys.argv[2]
+ofile  = sys.argv[3]
+
+b_ifile = ifile.encode('utf-8')
+b_ofile = ofile.encode('utf-8')
+
+if mode == "--encode":
+    lzss_functions.encode_file.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+    lzss_functions.encode_file(b_ifile, b_ofile)
+elif mode == "--decode":
+    lzss_functions.decode_file.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+    lzss_functions.decode_file(b_ifile, b_ofile)
+else:
+    print ("Error, invalid mode parameter, use --encode or --decode")