From ffa7ee9b67d048c9da9a8fff7b35ec2b52d30546 Mon Sep 17 00:00:00 2001 From: TerryE Date: Sun, 21 Feb 2016 04:18:27 +0000 Subject: [PATCH 1/2] Addition of encoder module with bas64 and hex encode/decode as per #925 --- app/include/user_modules.h | 1 + app/modules/crypto.c | 24 +++++- app/modules/encoder.c | 154 +++++++++++++++++++++++++++++++++++++ docs/en/modules/encoder.md | 75 ++++++++++++++++++ mkdocs.yml | 1 + 5 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 app/modules/encoder.c create mode 100644 docs/en/modules/encoder.md diff --git a/app/include/user_modules.h b/app/include/user_modules.h index 79e32a1fcb..6387f50881 100644 --- a/app/include/user_modules.h +++ b/app/include/user_modules.h @@ -22,6 +22,7 @@ #define LUA_USE_MODULES_COAP #define LUA_USE_MODULES_CRYPTO #define LUA_USE_MODULES_DHT +#define LUA_USE_MODULES_ENCODER //#define LUA_USE_MODULES_ENDUSER_SETUP // USE_DNS in dhcpserver.h needs to be enabled for this module to work. #define LUA_USE_MODULES_FILE #define LUA_USE_MODULES_GPIO diff --git a/app/modules/crypto.c b/app/modules/crypto.c index f3ae86f162..a7730191f1 100644 --- a/app/modules/crypto.c +++ b/app/modules/crypto.c @@ -37,7 +37,29 @@ static int crypto_sha1( lua_State* L ) return 1; } +#ifdef LUA_USE_MODULES_ENCODER +static int call_endcoder( lua_State* L, const char *function ) { + if (lua_gettop(L) != 1) { + luaL_error(L, "%s must have one argument", function); + } + lua_getfield(L, LUA_GLOBALSINDEX, "encoder"); + if (!lua_istable(L, -1) && !lua_isrotable(L, -1)) { // also need table just in case encoder has been overloaded + luaL_error(L, "Cannot find encoder.%s", function); + } + lua_getfield(L, -1, function); + lua_insert(L, 1); //move function below the argument + lua_pop(L, 1); //and dump the encoder rotable from stack. + lua_call(L,1,1); // call encoder.xxx(string) + return 1; +} +static int crypto_base64_encode (lua_State* L) { + return call_endcoder(L, "toBase64"); +} +static int crypto_hex_encode (lua_State* L) { + return call_endcoder(L, "toHex"); +} +#else static const char* bytes64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; /** * encoded = crypto.toBase64(raw) @@ -84,7 +106,7 @@ static int crypto_hex_encode( lua_State* L) c_free(out); return 1; } - +#endif /** * masked = crypto.mask(message, mask) * diff --git a/app/modules/encoder.c b/app/modules/encoder.c new file mode 100644 index 0000000000..523c1d6c19 --- /dev/null +++ b/app/modules/encoder.c @@ -0,0 +1,154 @@ +// Module encoders + +#include "module.h" +#include "lauxlib.h" +#include "lmem.h" +#include "c_string.h" +#define BASE64_INVALID '\xff' +#define BASE64_PADDING '=' +#define ISBASE64(c) (unbytes64[c] != BASE64_INVALID) + +static const char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +static char *toBase64 ( lua_State* L, const char *msg, size_t *len){ + size_t i, n = *len; + + if (!n) // handle empty string case + return NULL; + + char* q, *out = (char*)luaM_malloc(L, (n + 2) / 3 * 4); + char bytes64[sizeof(b64)]; + c_memcpy(bytes64, b64, sizeof(b64)); //Avoid lots of flash unaligned fetches + + for (i = 0, q = out; i < n; i += 3) { + int a = msg[i]; + int b = (i + 1 < n) ? msg[i + 1] : 0; + int c = (i + 2 < n) ? msg[i + 2] : 0; + *q++ = bytes64[a >> 2]; + *q++ = bytes64[((a & 3) << 4) | (b >> 4)]; + *q++ = (i + 1 < n) ? bytes64[((b & 15) << 2) | (c >> 6)] : BASE64_PADDING; + *q++ = (i + 2 < n) ? bytes64[(c & 63)] : BASE64_PADDING; + } + *len = q - out; + return out; +} + +static char *fromBase64 ( lua_State* L, const char *enc_msg, size_t *len){ + int i, n = *len, pad = 0; + unsigned const char *p; + unsigned char unbytes64[CHAR_MAX+1-CHAR_MIN], *msg, *q, a, b, c, d; + + if (!n) // handle empty string case + return NULL; + + if (n % 4) + luaL_error (L, "Invalid base64 string"); + + c_memset(unbytes64, BASE64_INVALID, sizeof(unbytes64)); + for (i = 0; i < sizeof(b64)-1; i++) unbytes64[b64[i]] = i; // sequential so no exceptions + + if (enc_msg[n-1] == BASE64_PADDING) { + pad = (enc_msg[n-2] != BASE64_PADDING) ? 1 : 2; + } + + for (i = 0; i < n - pad; i++) if (!ISBASE64(enc_msg[i])) luaL_error (L, "Invalid base64 string"); + unbytes64[BASE64_PADDING] = 0; + + msg = q = (unsigned char*) luaM_malloc(L, 1+ (3 * n / 4)); + for (i = 0, p = (unsigned char*) enc_msg;; i+=4) { + a = unbytes64[*p++], b = unbytes64[*p++], c = unbytes64[*p++], d = unbytes64[*p++] ; + if (i > n-4) break; + *q++ = (a << 2) | (b >> 4); + *q++ = (b << 4) | (c >> 2); + *q++ = (c << 6) | d; + } + + if (pad) { + *q++ = (a << 2) | (b >> 4); + if (pad == 1) *q++ = (b << 4) | (c >> 2); + } + *len = q - msg; + return (char *) msg; +} + +static inline char to_hex_nibble(unsigned char b) { + return b + ( b < 10 ? '0' : 'a' - 10 ); + } + +static char *toHex ( lua_State* L, const char *msg, size_t *len){ + int i, n = *len; + char *q, *out = (char*)luaM_malloc(L, n * 2); + for (i = 0, q = out; i < n; i++) { + *q++ = to_hex_nibble(msg[i] >> 4); + *q++ = to_hex_nibble(msg[i] & 0xf); + } + *len = 2*n; + return out; +} + +static char *fromHex ( lua_State* L, const char *msg, size_t *len){ + int i, n = *len; + const char *p; + char b, *q, *out = (char*)luaM_malloc(L, n * 2); + unsigned char c; + + if (n &1) + luaL_error (L, "Invalid hex string"); + + for (i = 0, p = msg, q = out; i < n; i++) { + if (*p >= '0' && *p <= '9') { + b = *p++ - '0'; + } else if (*p >= 'a' && *p <= 'f') { + b = *p++ - ('a' - 10); + } else if (*p >= 'A' && *p <= 'F') { + b = *p++ - ('A' - 10); + } else { + luaL_error (L, "Invalid hex string"); + } + if ((i&1) == 0) { + c = b<<4; + } else { + *q++ = c+ b; + } + } + *len = n>>1; + return out; +} + +// All encoder functions are of the form: +// Lua: output_string = encoder.function(input_string) +// Where input string maybe empty, but not nil +// Hence these all chare the do_func wrapper +static int do_func (lua_State *L, char * (*conv_func)(lua_State *, const char *, size_t *)) { + size_t l; + const char *input = luaL_checklstring(L, 1, &l); +// luaL_argcheck(L, l>0, 1, "input string empty"); + char *output = conv_func(L, input, &l); + + if (output) { + lua_pushlstring(L, output, l); + luaM_free(L, output); + } else { + lua_pushstring(L, ""); + } + return 1; +} + +#define DECLARE_FUNCTION(f) static int encoder_ ## f (lua_State *L) \ +{ return do_func(L, f); } + + DECLARE_FUNCTION(fromBase64); + DECLARE_FUNCTION(toBase64); + DECLARE_FUNCTION(fromHex); + DECLARE_FUNCTION(toHex); + +// Module function map +static const LUA_REG_TYPE encoder_map[] = { + { LSTRKEY("fromBase64"), LFUNCVAL(encoder_fromBase64) }, + { LSTRKEY("toBase64"), LFUNCVAL(encoder_toBase64) }, + { LSTRKEY("fromHex"), LFUNCVAL(encoder_fromHex) }, + { LSTRKEY("toHex"), LFUNCVAL(encoder_toHex) }, + { LNILKEY, LNILVAL } +}; + +NODEMCU_MODULE(ENCODER, "encoder", encoder_map, NULL); diff --git a/docs/en/modules/encoder.md b/docs/en/modules/encoder.md new file mode 100644 index 0000000000..a3135544b0 --- /dev/null +++ b/docs/en/modules/encoder.md @@ -0,0 +1,75 @@ +# encoder Module + +The encoder modules provides various functions for encoding and decoding byte data. + +## encoder.toBase64() + +Provides a Base64 representation of a (binary) Lua string. + +#### Syntax +`b64 = encoder.toBase64(binary)` + +#### Parameters +`binary` input string to Base64 encode + +#### Return +A Base64 encoded string. + +#### Example +```lua +print(encoder.toBase64(cyrpto.hash("sha1","abc"))) +``` + +## encoder.fromBase64() + +Decodes a Base64 representation of a (binary) Lua string back into the original string. + +#### Syntax +`binary_string = encoder.toBase64(b64)` + +#### Parameters +`b64` Base64 encoded input string + +#### Return +The decoded Lua (binary) string. + +#### Example +```lua +print(encoder.fromBase64(encoder.toBase64("hello world"))) +``` + +## encoder.toHex() + +Provides an ASCII hex representation of a (binary) Lua string. Each byte in the input string is represented as two hex characters in the output. + +#### Syntax +`hexstr = encoder.toHex(binary)` + +#### Parameters +`binary` input string to get hex representation for + +#### Returns +An ASCII hex string. + +#### Example +```lua +print(encoder.toHex(crypto.hash("sha1","abc"))) +``` + +## encoder.fromHex() + +Provides a Lua binary string decode of a ASCII hex string. Each byte in the output string is represented as two hex characters in the input. + +#### Syntax +`binary = encoder.toHex(hexstr)` + +#### Parameters +`hexstr` An ASCII hex string. + +#### Returns +Decoded string of hex representation. + +#### Example +```lua +print(encoder.fromHex("6a6a6a"))) +``` diff --git a/mkdocs.yml b/mkdocs.yml index ba7c42fa11..437fc0dd8e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -42,6 +42,7 @@ pages: - 'coap': 'en/modules/coap.md' - 'crypto': 'en/modules/crypto.md' - 'dht': 'en/modules/dht.md' + - 'encoder': 'en/modules/encoder.md' - 'enduser setup': 'en/modules/enduser-setup.md' - 'file': 'en/modules/file.md' - 'gpio': 'en/modules/gpio.md' From 8353991722d1c7c2cefc9b5eb271474c8b382f11 Mon Sep 17 00:00:00 2001 From: TerryE Date: Fri, 26 Feb 2016 00:44:33 +0000 Subject: [PATCH 2/2] Update encoder files after #925 & #1072 review comments --- app/modules/crypto.c | 6 ++-- app/modules/encoder.c | 59 +++++++++++++++++++++----------------- docs/en/modules/encoder.md | 14 +++++---- 3 files changed, 44 insertions(+), 35 deletions(-) diff --git a/app/modules/crypto.c b/app/modules/crypto.c index a7730191f1..fdfbbd37dc 100644 --- a/app/modules/crypto.c +++ b/app/modules/crypto.c @@ -38,7 +38,7 @@ static int crypto_sha1( lua_State* L ) } #ifdef LUA_USE_MODULES_ENCODER -static int call_endcoder( lua_State* L, const char *function ) { +static int call_encoder( lua_State* L, const char *function ) { if (lua_gettop(L) != 1) { luaL_error(L, "%s must have one argument", function); } @@ -54,10 +54,10 @@ static int call_endcoder( lua_State* L, const char *function ) { } static int crypto_base64_encode (lua_State* L) { - return call_endcoder(L, "toBase64"); + return call_encoder(L, "toBase64"); } static int crypto_hex_encode (lua_State* L) { - return call_endcoder(L, "toHex"); + return call_encoder(L, "toHex"); } #else static const char* bytes64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; diff --git a/app/modules/encoder.c b/app/modules/encoder.c index 523c1d6c19..9590627a2a 100644 --- a/app/modules/encoder.c +++ b/app/modules/encoder.c @@ -8,16 +8,16 @@ #define BASE64_PADDING '=' #define ISBASE64(c) (unbytes64[c] != BASE64_INVALID) -static const char b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const uint8 b64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; -static char *toBase64 ( lua_State* L, const char *msg, size_t *len){ +static uint8 *toBase64 ( lua_State* L, const uint8 *msg, size_t *len){ size_t i, n = *len; if (!n) // handle empty string case return NULL; - char* q, *out = (char*)luaM_malloc(L, (n + 2) / 3 * 4); - char bytes64[sizeof(b64)]; + uint8 * q, *out = (uint8 *)luaM_malloc(L, (n + 2) / 3 * 4); + uint8 bytes64[sizeof(b64)]; c_memcpy(bytes64, b64, sizeof(b64)); //Avoid lots of flash unaligned fetches for (i = 0, q = out; i < n; i += 3) { @@ -33,15 +33,15 @@ static char *toBase64 ( lua_State* L, const char *msg, size_t *len){ return out; } -static char *fromBase64 ( lua_State* L, const char *enc_msg, size_t *len){ - int i, n = *len, pad = 0; - unsigned const char *p; - unsigned char unbytes64[CHAR_MAX+1-CHAR_MIN], *msg, *q, a, b, c, d; +static uint8 *fromBase64 ( lua_State* L, const uint8 *enc_msg, size_t *len){ + int i, n = *len, blocks = (n>>2), pad = 0; + const uint8 *p; + uint8 unbytes64[UCHAR_MAX+1], *msg, *q; if (!n) // handle empty string case return NULL; - if (n % 4) + if (n & 3) luaL_error (L, "Invalid base64 string"); c_memset(unbytes64, BASE64_INVALID, sizeof(unbytes64)); @@ -49,35 +49,40 @@ static char *fromBase64 ( lua_State* L, const char *enc_msg, size_t *len){ if (enc_msg[n-1] == BASE64_PADDING) { pad = (enc_msg[n-2] != BASE64_PADDING) ? 1 : 2; + blocks--; //exclude padding block } for (i = 0; i < n - pad; i++) if (!ISBASE64(enc_msg[i])) luaL_error (L, "Invalid base64 string"); unbytes64[BASE64_PADDING] = 0; - msg = q = (unsigned char*) luaM_malloc(L, 1+ (3 * n / 4)); - for (i = 0, p = (unsigned char*) enc_msg;; i+=4) { - a = unbytes64[*p++], b = unbytes64[*p++], c = unbytes64[*p++], d = unbytes64[*p++] ; - if (i > n-4) break; + msg = q = (uint8 *) luaM_malloc(L, 1+ (3 * n / 4)); + for (i = 0, p = enc_msg; i> 4); *q++ = (b << 4) | (c >> 2); *q++ = (c << 6) | d; } - if (pad) { + if (pad) { //now process padding block bytes + uint8 a = unbytes64[*p++]; + uint8 b = unbytes64[*p++]; *q++ = (a << 2) | (b >> 4); - if (pad == 1) *q++ = (b << 4) | (c >> 2); + if (pad == 1) *q++ = (b << 4) | (unbytes64[*p] >> 2); } *len = q - msg; - return (char *) msg; + return msg; } -static inline char to_hex_nibble(unsigned char b) { +static inline uint8 to_hex_nibble(uint8 b) { return b + ( b < 10 ? '0' : 'a' - 10 ); } -static char *toHex ( lua_State* L, const char *msg, size_t *len){ +static uint8 *toHex ( lua_State* L, const uint8 *msg, size_t *len){ int i, n = *len; - char *q, *out = (char*)luaM_malloc(L, n * 2); + uint8 *q, *out = (uint8 *)luaM_malloc(L, n * 2); for (i = 0, q = out; i < n; i++) { *q++ = to_hex_nibble(msg[i] >> 4); *q++ = to_hex_nibble(msg[i] & 0xf); @@ -86,11 +91,11 @@ static char *toHex ( lua_State* L, const char *msg, size_t *len){ return out; } -static char *fromHex ( lua_State* L, const char *msg, size_t *len){ +static uint8 *fromHex ( lua_State* L, const uint8 *msg, size_t *len){ int i, n = *len; - const char *p; - char b, *q, *out = (char*)luaM_malloc(L, n * 2); - unsigned char c; + const uint8 *p; + uint8 b, *q, *out = (uint8 *)luaM_malloc(L, n * 2); + uint8 c; if (n &1) luaL_error (L, "Invalid hex string"); @@ -118,12 +123,12 @@ static char *fromHex ( lua_State* L, const char *msg, size_t *len){ // All encoder functions are of the form: // Lua: output_string = encoder.function(input_string) // Where input string maybe empty, but not nil -// Hence these all chare the do_func wrapper -static int do_func (lua_State *L, char * (*conv_func)(lua_State *, const char *, size_t *)) { +// Hence these all call the do_func wrapper +static int do_func (lua_State *L, uint8 * (*conv_func)(lua_State *, const uint8 *, size_t *)) { size_t l; - const char *input = luaL_checklstring(L, 1, &l); + const uint8 *input = luaL_checklstring(L, 1, &l); // luaL_argcheck(L, l>0, 1, "input string empty"); - char *output = conv_func(L, input, &l); + uint8 *output = conv_func(L, input, &l); if (output) { lua_pushlstring(L, output, l); diff --git a/docs/en/modules/encoder.md b/docs/en/modules/encoder.md index a3135544b0..a57bc6b3a7 100644 --- a/docs/en/modules/encoder.md +++ b/docs/en/modules/encoder.md @@ -17,12 +17,13 @@ A Base64 encoded string. #### Example ```lua -print(encoder.toBase64(cyrpto.hash("sha1","abc"))) +print(encoder.toBase64(crypto.hash("sha1","abc"))) ``` ## encoder.fromBase64() -Decodes a Base64 representation of a (binary) Lua string back into the original string. +Decodes a Base64 representation of a (binary) Lua string back into the original string. An error is +thrown if the string is not a valid base64 encoding. #### Syntax `binary_string = encoder.toBase64(b64)` @@ -40,7 +41,8 @@ print(encoder.fromBase64(encoder.toBase64("hello world"))) ## encoder.toHex() -Provides an ASCII hex representation of a (binary) Lua string. Each byte in the input string is represented as two hex characters in the output. +Provides an ASCII hex representation of a (binary) Lua string. Each byte in the input string is +represented as two hex characters in the output. #### Syntax `hexstr = encoder.toHex(binary)` @@ -58,10 +60,12 @@ print(encoder.toHex(crypto.hash("sha1","abc"))) ## encoder.fromHex() -Provides a Lua binary string decode of a ASCII hex string. Each byte in the output string is represented as two hex characters in the input. +Returns the Lua binary string decode of a ASCII hex string. Each byte in the output string is +represented as two hex characters in the input. An error is thrown if the string is not a +valid base64 encoding. #### Syntax -`binary = encoder.toHex(hexstr)` +`binary = encoder.fromHex(hexstr)` #### Parameters `hexstr` An ASCII hex string.