lua fix mem_read_write, strtol

back when I wrote this I was running the sim on a 32 bit machine
I didn't catch the hardcoded LONG_MAX reference or the fact that
lua_tointeger maxes ot at 32 bits

on 64 bit machines strtol caused all kinds of issues especially since
it returned the real LONG_MIN/MAX values

Change-Id: I3571ebbd9df333f7cbf4077562412c27429bfadc

2 files changed, 4 insertions, 3 deletions

diff --git a/apps/plugins/lua/rocklib.c b/apps/plugins/lua/rocklib.c
index 8c662b7359..3c38440850 100644
--- a/apps/plugins/lua/rocklib.c
+++ b/apps/plugins/lua/rocklib.c
@@ -642,8 +642,8 @@ RB_WRAP(strncasecmp)
 
 static int mem_read_write(lua_State *L, uintptr_t address, size_t maxsize)
 {
-    intptr_t offset = (intptr_t) luaL_optint(L, 1, 0);
-    size_t   size   = (size_t)   luaL_optint(L, 2, maxsize);
+    intptr_t offset = (intptr_t) luaL_optnumber(L, 1, 0);
+    size_t   size   = (size_t)   luaL_optnumber(L, 2, maxsize);
     size_t   written;
     int      type   = lua_type(L, 3);
 
@@ -651,6 +651,7 @@ static int mem_read_write(lua_State *L, uintptr_t address, size_t maxsize)
     {
         /* allows pointer within structure to be calculated offset */
         offset = -(address + offset);
+        luaL_argcheck(L, ((size_t) offset) <= maxsize,  1, ERR_IDX_RANGE);
         size   = (size_t) maxsize - offset;
     }
 
diff --git a/apps/plugins/lua/strtol.c b/apps/plugins/lua/strtol.c
index 184951b844..3dd29b6b57 100644
--- a/apps/plugins/lua/strtol.c
+++ b/apps/plugins/lua/strtol.c
@@ -2,7 +2,7 @@
 
 extern unsigned long int strtoul(const char *ptr, char **endptr, int base);
 
-#define ABS_LONG_MIN 2147483648UL
+#define ABS_LONG_MIN LONG_MAX
 long int strtol(const char *nptr, char **endptr, int base)
 {
   int neg=0;