Optimise ROTable accesses and interface (#2505)

-  Optimise ROTable accesses and interface

This includes some refinements to the ROTable cache which remove the linker cludges on the CROSS_COMPILE builds.  Also keyhole tweaks to some of the Lua VM code to implrove runtimes.

I also noticed some compile time warnings during the build; the change to uz_unzip.c doesn't impact the compiled code, but does remove the compiler warnings.

app/include/module.h

@@ -57,18 +57,8 @@
   const LOCK_IN_SECTION(libs) \
     luaL_Reg MODULE_PASTE_(lua_lib_,cfgname) = { luaname, initfunc }; \
   const LOCK_IN_SECTION(rotable) \
-    luaR_table MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(LUA_USE_MODULES_,cfgname))) \
-    = { luaname, map }
-
-
-/* System module registration support, not using LUA_USE_MODULES_XYZ. */
-#define BUILTIN_LIB_INIT(name, luaname, initfunc) \
-  const LOCK_IN_SECTION(libs) \
-    luaL_Reg MODULE_PASTE_(lua_lib_,name) = { luaname, initfunc }
-
-#define BUILTIN_LIB(name, luaname, map) \
-  const LOCK_IN_SECTION(rotable) \
-    luaR_table MODULE_PASTE_(lua_rotable_,name) = { luaname, map }
+    luaR_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(LUA_USE_MODULES_,cfgname))) \
+    = {LSTRKEY(luaname), LROVAL(map)}
 
 #if !defined(LUA_CROSS_COMPILER) && !(MIN_OPT_LEVEL==2 && LUA_OPTIMIZE_MEMORY==2)
 # error "NodeMCU modules must be built with LTR enabled (MIN_OPT_LEVEL=2 and LUA_OPTIMIZE_MEMORY=2)"

app/lua/lauxlib.c

@@ -25,8 +25,6 @@
 #define lauxlib_c
 #define LUA_LIB
 
-#include "lrotable.h"
-
 #include "lauxlib.h"
 #include "lgc.h"
 #include "ldo.h"
@@ -555,14 +553,7 @@ LUALIB_API const char *luaL_findtable (lua_State *L, int idx,
     if (e == NULL) e = fname + c_strlen(fname);
     lua_pushlstring(L, fname, e - fname);
     lua_rawget(L, -2);
-    if (lua_isnil(L, -1)) {
-      /* If looking for a global variable, check the rotables too */
-      void *ptable = luaR_findglobal(fname, e - fname);
-      if (ptable) {
-        lua_pop(L, 1);
-        lua_pushrotable(L, ptable);
-      }
-    }
+
     if (lua_isnil(L, -1)) {  /* no such field? */
       lua_pop(L, 1);  /* remove this nil */
       lua_createtable(L, 0, (*e == '.' ? 1 : szhint)); /* new table for field */

app/lua/lbaselib.c

@@ -16,7 +16,7 @@
 #include C_HEADER_STDLIB
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lrotable.h"
+#include "lrodefs.h"
 
 
 
@@ -462,71 +462,62 @@ static int luaB_newproxy (lua_State *L) {
   return 1;
 }
 
-#define LUA_BASELIB_FUNCLIST\
-  {LSTRKEY("assert"), LFUNCVAL(luaB_assert)},\
-  {LSTRKEY("collectgarbage"), LFUNCVAL(luaB_collectgarbage)},\
-  {LSTRKEY("dofile"), LFUNCVAL(luaB_dofile)},\
-  {LSTRKEY("error"), LFUNCVAL(luaB_error)},\
-  {LSTRKEY("gcinfo"), LFUNCVAL(luaB_gcinfo)},\
-  {LSTRKEY("getfenv"), LFUNCVAL(luaB_getfenv)},\
-  {LSTRKEY("getmetatable"), LFUNCVAL(luaB_getmetatable)},\
-  {LSTRKEY("loadfile"), LFUNCVAL(luaB_loadfile)},\
-  {LSTRKEY("load"), LFUNCVAL(luaB_load)},\
-  {LSTRKEY("loadstring"), LFUNCVAL(luaB_loadstring)},\
-  {LSTRKEY("next"), LFUNCVAL(luaB_next)},\
-  {LSTRKEY("pcall"), LFUNCVAL(luaB_pcall)},\
-  {LSTRKEY("print"), LFUNCVAL(luaB_print)},\
-  {LSTRKEY("rawequal"), LFUNCVAL(luaB_rawequal)},\
-  {LSTRKEY("rawget"), LFUNCVAL(luaB_rawget)},\
-  {LSTRKEY("rawset"), LFUNCVAL(luaB_rawset)},\
-  {LSTRKEY("select"), LFUNCVAL(luaB_select)},\
-  {LSTRKEY("setfenv"), LFUNCVAL(luaB_setfenv)},\
-  {LSTRKEY("setmetatable"), LFUNCVAL(luaB_setmetatable)},\
-  {LSTRKEY("tonumber"), LFUNCVAL(luaB_tonumber)},\
-  {LSTRKEY("tostring"), LFUNCVAL(luaB_tostring)},\
-  {LSTRKEY("type"), LFUNCVAL(luaB_type)},\
-  {LSTRKEY("unpack"), LFUNCVAL(luaB_unpack)},\
-  {LSTRKEY("xpcall"), LFUNCVAL(luaB_xpcall)}
-
-#if LUA_OPTIMIZE_MEMORY == 2
-#undef MIN_OPT_LEVEL
-#define MIN_OPT_LEVEL 2
 #include "lrodefs.h"
-const LUA_REG_TYPE base_funcs_list[] = {
-  LUA_BASELIB_FUNCLIST,
-  {LNILKEY, LNILVAL}
-};
-#endif
 
+extern const luaR_entry lua_rotable_base[];
 
-static int luaB_index(lua_State *L) {
-#if LUA_OPTIMIZE_MEMORY == 2
-  int fres;
-  if ((fres = luaR_findfunction(L, base_funcs_list)) != 0)
-    return fres;
-#endif  
-  const char *keyname = luaL_checkstring(L, 2);
-  if (!c_strcmp(keyname, "_VERSION")) {
-    lua_pushliteral(L, LUA_VERSION);
-    return 1;
-  }
-  void *res = luaR_findglobal(keyname, c_strlen(keyname));
-  if (!res)
-    return 0;
-  else {
-    lua_pushrotable(L, res);
-    return 1;
-  }
-}
+/*
+ * ESP builds use specific linker directives to marshal all ROTable declarations
+ * into a single ROTable in the PSECT ".lua_rotable".  
+ *
+ * This is not practical on Posix builds using a standard link so for cross
+ * compiler builds, separate ROTables are used for the base functions and library
+ * ROTables, with the latter chained from the former using its __index meta-method.
+ * In this case all library ROTables are defined in linit.c.
+ */
+#ifdef LUA_CROSS_COMPILER
+#define BASE_ROTABLE    base_func_map
+#define LOCK_IN_ROTABLE
+static const LUA_REG_TYPE base_func_meta[] = {
+  LROT_TABENTRY(__index, lua_rotable_base),     
+  LROT_END};
+#else
+#define BASE_ROTABLE    lua_rotable_base
+#define LOCK_IN_ROTABLE __attribute__((used,unused,section(".lua_rotable")))
+#endif
 
-static const luaL_Reg base_funcs[] = {
-#if LUA_OPTIMIZE_MEMORY != 2
-#undef MIN_OPT_LEVEL
-#define MIN_OPT_LEVEL 0
-#include "lrodefs.h"
-  LUA_BASELIB_FUNCLIST,
+static const LUA_REG_TYPE LOCK_IN_ROTABLE base_func_map[] = {
+  LROT_FUNCENTRY(assert,         luaB_assert),     
+  LROT_FUNCENTRY(collectgarbage, luaB_collectgarbage),     
+  LROT_FUNCENTRY(dofile,         luaB_dofile),     
+  LROT_FUNCENTRY(error,          luaB_error),     
+  LROT_FUNCENTRY(gcinfo,         luaB_gcinfo),     
+  LROT_FUNCENTRY(getfenv,        luaB_getfenv),     
+  LROT_FUNCENTRY(getmetatable,   luaB_getmetatable),     
+  LROT_FUNCENTRY(loadfile,       luaB_loadfile),     
+  LROT_FUNCENTRY(load,           luaB_load),     
+  LROT_FUNCENTRY(loadstring,     luaB_loadstring),     
+  LROT_FUNCENTRY(next,           luaB_next),     
+  LROT_FUNCENTRY(pcall,          luaB_pcall),     
+  LROT_FUNCENTRY(print,          luaB_print),     
+  LROT_FUNCENTRY(rawequal,       luaB_rawequal),     
+  LROT_FUNCENTRY(rawget,         luaB_rawget),     
+  LROT_FUNCENTRY(rawset,         luaB_rawset),     
+  LROT_FUNCENTRY(select,         luaB_select),     
+  LROT_FUNCENTRY(setfenv,        luaB_setfenv),     
+  LROT_FUNCENTRY(setmetatable,   luaB_setmetatable),     
+  LROT_FUNCENTRY(tonumber,       luaB_tonumber),     
+  LROT_FUNCENTRY(tostring,       luaB_tostring),     
+  LROT_FUNCENTRY(type,           luaB_type),     
+  LROT_FUNCENTRY(unpack,         luaB_unpack),     
+  LROT_FUNCENTRY(xpcall,         luaB_xpcall)     
+#ifdef LUA_CROSS_COMPILER
+ ,LROT_TABENTRY(__metatable,     base_func_meta),   
+  LROT_END
 #endif
-  {"__index", luaB_index},
+};
+
+static const luaL_Reg base_funcs[] = {
   {NULL, NULL}
 };
 
@@ -661,7 +652,6 @@ static int luaB_corunning (lua_State *L) {
 
 #undef MIN_OPT_LEVEL
 #define MIN_OPT_LEVEL 1
-#include "lrodefs.h"
 const LUA_REG_TYPE co_funcs[] = {
   {LSTRKEY("create"), LFUNCVAL(luaB_cocreate)},
   {LSTRKEY("resume"), LFUNCVAL(luaB_coresume)},
@@ -682,7 +672,6 @@ static void auxopen (lua_State *L, const char *name,
   lua_setfield(L, -2, name);
 }
 
-
 static void base_open (lua_State *L) {
   /* set global _G */
   lua_pushvalue(L, LUA_GLOBALSINDEX);
@@ -691,11 +680,12 @@ static void base_open (lua_State *L) {
   luaL_register_light(L, "_G", base_funcs);
 #if LUA_OPTIMIZE_MEMORY > 0
   lua_pushvalue(L, -1);
-  lua_setmetatable(L, -2);  
-#else
+  lua_setmetatable(L, -2);
+  lua_pushrotable(L, (void *)BASE_ROTABLE);
+  lua_setglobal(L, "__index");
+#endif
   lua_pushliteral(L, LUA_VERSION);
   lua_setglobal(L, "_VERSION");  /* set global _VERSION */
-#endif
   /* `ipairs' and `pairs' need auxliliary functions as upvalues */
   auxopen(L, "ipairs", luaB_ipairs, ipairsaux);
   auxopen(L, "pairs", luaB_pairs, luaB_next);

app/lua/ldblib.c

@@ -16,7 +16,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lrotable.h"
 #include "lstring.h"
 #include "lflash.h"
 #include "user_modules.h"

app/lua/lgc.c

@@ -168,10 +168,14 @@ static int traversetable (global_State *g, Table *h) {
   int i;
   int weakkey = 0;
   int weakvalue = 0;
-  const TValue *mode;
-  if (h->metatable && !luaR_isrotable(h->metatable))
-    markobject(g, h->metatable);
-  mode = gfasttm(g, h->metatable, TM_MODE);
+  const TValue *mode = luaO_nilobject;
+
+  if (h->metatable) {
+    if (!luaR_isrotable(h->metatable))
+      markobject(g, h->metatable);
+    mode = gfasttm(g, h->metatable, TM_MODE);
+  }
+
   if (mode && ttisstring(mode)) {  /* is there a weak mode? */
     weakkey = (c_strchr(svalue(mode), 'k') != NULL);
     weakvalue = (c_strchr(svalue(mode), 'v') != NULL);

app/lua/linit.c

@@ -15,60 +15,71 @@
 #include "lauxlib.h"
 #include "luaconf.h"
 #include "module.h"
-#if defined(LUA_CROSS_COMPILER)
-BUILTIN_LIB(      start_list,  NULL,               NULL);
-BUILTIN_LIB_INIT( start_list,  NULL,               NULL);
+
+#if !defined(LUA_CROSS_COMPILER) && !(MIN_OPT_LEVEL==2 && LUA_OPTIMIZE_MEMORY==2)
+# error "NodeMCU modules must be built with LTR enabled (MIN_OPT_LEVEL=2 and LUA_OPTIMIZE_MEMORY=2)"
 #endif
+
 extern const luaR_entry strlib[], tab_funcs[],  dblib[], 
                         co_funcs[], math_map[], syslib[];
+extern const luaR_entry syslib[], io_funcs[];  // Only used on cross-compile builds
 
-BUILTIN_LIB_INIT( BASE,      "",                luaopen_base);
-BUILTIN_LIB_INIT( LOADLIB,   LUA_LOADLIBNAME,   luaopen_package);
-
-BUILTIN_LIB(      STRING,    LUA_STRLIBNAME,    strlib);
-BUILTIN_LIB_INIT( STRING,    LUA_STRLIBNAME,    luaopen_string);
-
-BUILTIN_LIB(      TABLE,     LUA_TABLIBNAME,    tab_funcs);
-BUILTIN_LIB_INIT( TABLE,     LUA_TABLIBNAME,    luaopen_table);
-
-BUILTIN_LIB(      DBG,       LUA_DBLIBNAME,     dblib);
-BUILTIN_LIB_INIT( DBG,       LUA_DBLIBNAME,     luaopen_debug);
-
-BUILTIN_LIB(      CO,        LUA_COLIBNAME,     co_funcs);
-
-BUILTIN_LIB(      MATH,      LUA_MATHLIBNAME,   math_map);
-
-#if defined(LUA_CROSS_COMPILER)
-extern const luaR_entry syslib[], iolib[];
-BUILTIN_LIB(      OS,        LUA_OSLIBNAME,     syslib);
-BUILTIN_LIB_INIT( IO,        LUA_IOLIBNAME,     luaopen_io);
-BUILTIN_LIB(      end_list,  NULL,               NULL);
-BUILTIN_LIB_INIT( end_list,  NULL,               NULL);
 /*
- * These base addresses are internal to this module for cross compile builds
- * This also exploits feature of the GCC code generator that the variables are
- * emitted in either normal OR reverse order within PSECT.
+ * The NodeMCU Lua initalisation has been adapted to use linker-based module 
+ * registration.  This uses a PSECT naming convention to allow the lib and rotab
+ * entries to be collected by the linker into consoliated tables.  The linker
+ * defines lua_libs_base and lua_rotable_base.
+ *
+ * This is not practical on Posix builds which use a standard loader declaration
+ * so for cross compiler builds, separate ROTables are used for the base functions
+ * and library ROTables, with the latter chained from the former using its __index 
+ * meta-method. In this case all library ROTables are defined here, avoiding the
+ * need for linker magic is avoided on host builds. 
  */
-#define isascending(n) ((&(n ## _end_list)-&(n ## _start_list))>0)
-static const luaL_Reg   *lua_libs;
-const        luaR_table *lua_rotable;
-#else 
-/* These base addresses are Xtensa toolchain linker constants for Firmware builds */ 
+
+#if defined(LUA_CROSS_COMPILER)
+#define LUA_ROTABLES lua_rotable_base
+#define LUA_LIBS     lua_libs_base
+#else /* declare Xtensa toolchain linker defined constants */ 
 extern const luaL_Reg    lua_libs_base[];
-extern const luaR_table  lua_rotable_base[];
-static const luaL_Reg   *lua_libs    = lua_libs_base;
-const        luaR_table *lua_rotable = lua_rotable_base;
+extern const luaR_entry  lua_rotable_base[];
+#define LUA_ROTABLES lua_rotable_core
+#define LUA_LIBS     lua_libs_core
 #endif
+
+static const LOCK_IN_SECTION(libs) luaL_reg LUA_LIBS[] = {
+  {"",              luaopen_base},
+  {LUA_LOADLIBNAME, luaopen_package},
+  {LUA_STRLIBNAME,  luaopen_string},
+  {LUA_TABLIBNAME,  luaopen_table},
+  {LUA_DBLIBNAME,   luaopen_debug}
+#if defined(LUA_CROSS_COMPILER)
+ ,{LUA_IOLIBNAME,   luaopen_io},
+  {NULL,            NULL}
+#endif
+};
 
-void luaL_openlibs (lua_State *L) {
+#define ENTRY(n,t)  {LSTRKEY(n), LRO_ROVAL(t)}
+
+const LOCK_IN_SECTION(rotable) ROTable LUA_ROTABLES[] = {
+  ENTRY("ROM",           LUA_ROTABLES),
+  ENTRY(LUA_STRLIBNAME,  strlib),
+  ENTRY(LUA_TABLIBNAME,  tab_funcs),
+  ENTRY(LUA_DBLIBNAME,   dblib),
+  ENTRY(LUA_COLIBNAME,   co_funcs),
+  ENTRY(LUA_MATHLIBNAME, math_map)
 #if defined(LUA_CROSS_COMPILER)
-lua_libs    = (isascending(lua_lib) ? &lua_lib_start_list : &lua_lib_end_list) + 1;
-lua_rotable = (isascending(lua_rotable) ? &lua_rotable_start_list : &lua_rotable_end_list) + 1;
+ ,ENTRY(LUA_OSLIBNAME,   syslib),
+  LROT_END
 #endif
-  const luaL_Reg *lib = lua_libs;
+  };
+
+void luaL_openlibs (lua_State *L) {
+  const luaL_Reg *lib = lua_libs_base;
+
+  /* loop round and open libraries */
   for (; lib->name; lib++) {
-    if (lib->func)
-    {
+    if (lib->func) {
       lua_pushcfunction(L, lib->func);
       lua_pushstring(L, lib->name);
       lua_call(L, 1, 0);

app/lua/lmathlib.c

@@ -15,7 +15,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lrotable.h"
 
 #undef PI
 #define PI (3.14159265358979323846)

app/lua/loadlib.c

@@ -24,7 +24,6 @@
 
 #include "lauxlib.h"
 #include "lualib.h"
-#include "lrotable.h"
 
 /* prefix for open functions in C libraries */
 #define LUA_POF		"luaopen_"
@@ -474,10 +473,11 @@ static int ll_require (lua_State *L) {
     return 1;  /* package is already loaded */
   }
   /* Is this a readonly table? */
-  void *res = luaR_findglobal(name, c_strlen(name));
-  if (res) {
-    lua_pushrotable(L, res);
+  lua_getfield(L, LUA_GLOBALSINDEX, name);
+  if(lua_isrotable(L,-1)) {
     return 1;
+  } else {
+    lua_pop(L, 1);
   }
   /* else must load it; iterate over available loaders */
   lua_getfield(L, LUA_ENVIRONINDEX, "loaders");
@@ -563,8 +563,13 @@ static void modinit (lua_State *L, const char *modname) {
 
 static int ll_module (lua_State *L) {
   const char *modname = luaL_checkstring(L, 1);
-  if (luaR_findglobal(modname, c_strlen(modname)))
+  /* Is this a readonly table? */
+  lua_getfield(L, LUA_GLOBALSINDEX, modname);
+  if(lua_isrotable(L,-1)) {
     return 0;
+  } else {
+    lua_pop(L, 1);
+  }
   int loaded = lua_gettop(L) + 1;  /* index of _LOADED table */
   lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED");
   lua_getfield(L, loaded, modname);  /* get _LOADED[modname] */