lua and luac macro fixes and new "include" directive

Each lua file has it's own `MACRO` table which is cleared at the
beginning of parsing.
To support import of macros from other files "include" directive can be
used as `include "filename"` to load content of the file at the
compilation step, for example:
`include "macro_functions"`
will try to open the file "macro_functions", load it's content and paste
it at the place of `include` directive. Of course, any content of any
file can be included this way, just like in C preprocessor.

lua/src/llex.c

@@ -36,7 +36,7 @@ const char *const luaX_tokens [] = {
   "and", "break", "do", "else", "elseif",
   "end", "false", "for", "function", "if",
   "in", "local", "nil", "not", "or", "repeat",
-  "return", "then", "true", "until", "while",
+  "return", "then", "true", "until", "include", "while",
   "..", "...", "==", ">=", "<=", "~=",
   "<<", ">>", "//",
   "<number>", "<name>", "<string>", "<eof>",
@@ -144,6 +144,10 @@ void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) {
   ls->linenumber = 1;
   ls->lastline = 1;
   ls->source = source;
+
+  lua_pushnil(L);
+  lua_setglobal(L, MACRO);
+
   luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER);  /* initialize buffer */
   next(ls);  /* read first char */
 }
@@ -332,6 +336,32 @@ static void read_string (LexState *ls, int del, SemInfo *seminfo) {
                                luaZ_bufflen(ls->buff) - 2);
 }
 
+static int insert_string(LexState *ls, const char* s) {
+    luaZ_resetbuffer(ls->buff);
+    if (ls->mpos < ls->mlen) {
+      int len = strlen(s) + ls->mlen - ls->mpos + 1;
+      char *res = (char*) malloc(len);
+      strcpy(res, s);
+      strcat(res, &ls->mstr[ls->mpos-1]);
+      strcat(res, " ");
+      free(ls->mstr);
+      ls->mstr = res;
+      ls->z->n--; ls->z->p++;
+    }
+    else {
+      if (ls->mlen > 0) free(ls->mstr);
+      char *res = (char*) malloc(strlen(s)+1);
+      strcpy(res, s);
+      ls->mstr = res;
+    }
+    ls->mpos = 1;
+    ls->mlen = strlen(ls->mstr);
+    if (ls->mlen > 0) {
+      ls->current = ls->mstr[0];
+      ls->z->n++; ls->z->p--;
+    }
+    return 0;
+}
 
 static int llex (LexState *ls, SemInfo *seminfo) {
   luaZ_resetbuffer(ls->buff);
@@ -466,12 +496,14 @@ static int llex (LexState *ls, SemInfo *seminfo) {
           else luaX_lexerror(ls, "cannot allocate enough memory",0);
           lua_pushstring(L, key);
           lua_gettable(L, -2);
+
           if (!lua_isnil(L, -1))
           {
             if (ls->current != '@')
               luaX_lexerror(ls, "definition of existent macro", 0);
           } else if (ls->current == '@')
             luaX_lexerror(ls, "redefinition of nonexistent macro", 0);
+
           if (ls->current == '@') next(ls);
           lua_pop(L, 1);
           lua_pushstring(L, key);
@@ -618,8 +650,40 @@ static int llex (LexState *ls, SemInfo *seminfo) {
           free(val);
           continue;
         }
-        if (ts->tsv.reserved > 0)  /* reserved word? */
+        if (ts->tsv.reserved > 0) {  /* reserved word? */
+          if (ts->tsv.reserved == TK_INCLUDE - FIRST_RESERVED + 1) {
+              int t = llex(ls, seminfo);
+              if (t == TK_STRING) {
+                  const char *filename = getstr(seminfo->ts);
+                  char * buffer = 0;
+                  long length;
+                  FILE * f = fopen (filename, "rb");
+
+                  if (f)
+                  {
+                    fseek (f, 0, SEEK_END);
+                    length = ftell (f);
+                    fseek (f, 0, SEEK_SET);
+                    buffer = malloc (length + 1);
+                    if (buffer)
+                    {
+                      fread (buffer, 1, length, f);
+                      buffer[length] = '\0';
+                    }
+                    fclose (f);
+                  }
+
+                  if (buffer)
+                  {
+                    ++ls->mpos;
+                    insert_string(ls, buffer);
+                    continue;
+                  }
+                  luaX_lexerror(ls, "file not found", TK_INCLUDE);
+              } else luaX_lexerror(ls, "string expected", TK_INCLUDE);
+          }
           return ts->tsv.reserved - 1 + FIRST_RESERVED;
+        }
         lua_State *L = ls->L;
         lua_getglobal(L, MACRO);
         if (lua_istable(L, -1)) {
@@ -717,39 +781,18 @@ static int llex (LexState *ls, SemInfo *seminfo) {
           }
 
           if (lua_isstring(L, -1)) {
-            luaZ_resetbuffer(ls->buff);
             const char *s = lua_tostring(L, -1);
-            if (ls->mpos < ls->mlen) {
-              int len = strlen(s) + ls->mlen - ls->mpos + 1;
-              char *res = (char*) malloc(len);
-              strcpy(res, s);
-              strcat(res, &ls->mstr[ls->mpos-1]);
-              strcat(res, " ");
-              free(ls->mstr);
-              ls->mstr = res;
-              ls->z->n--; ls->z->p++;
-            }
-            else {
-              if (ls->mlen > 0) free(ls->mstr);
-              char *res = (char*) malloc(strlen(s)+1);
-              strcpy(res, s);
-              ls->mstr = res;
-            }
-            ls->mpos = 1;
-            ls->mlen = strlen(ls->mstr);
-            if (ls->mlen > 0) {
-              ls->current = ls->mstr[0];
-              ls->z->n++; ls->z->p--;
-            }
+            insert_string(ls, s);
             lua_pop(L, 1);
             lua_pop(L, 1);
             if (ls->mswt++ > 1e6) luaX_lexerror(ls, "possible mutual macro recursion", 0);
             continue;
           }
+
           lua_pop(L, 1);
         }
-        lua_pop(L, 1);
 
+        lua_pop(L, 1);
         seminfo->ts = ts;
         return TK_NAME;
       }

lua/src/llex.h

@@ -26,7 +26,7 @@ enum RESERVED {
   TK_AND = FIRST_RESERVED, TK_BREAK,
   TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION,
   TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT,
-  TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE,
+  TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_INCLUDE, TK_WHILE,
   /* other terminal symbols */
   TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE,
   TK_LSHFT, TK_RSHFT, TK_INTDIV,

lua/src/luac.c

@@ -25,6 +25,17 @@
 #include "lzio.c"
 
 #include "lauxlib.c"
+#include "lbaselib.c"
+#include "ldblib.c"
+#include "liolib.c"
+#include "lmathlib.c"
+#include "loadlib.c"
+#include "luacoslib.c"
+#include "lstrlib.c"
+#include "ltablib.c"
+#include "lutf8lib.c"
+#include "lint64.c"
+#include "linit.c"
 #include "print.c"
 
 #include <errno.h>
@@ -167,7 +178,7 @@ static const Proto* combine(lua_State* L, int n)
  }
 }
 
-static int writer(lua_State* L, const void* p, size_t size, void* u)
+static int luac_writer(lua_State* L, const void* p, size_t size, void* u)
 {
  UNUSED(L);
  return (fwrite(p,size,1,(FILE*)u)!=1) && (size!=0);
@@ -198,7 +209,7 @@ static int pmain(lua_State* L)
   FILE* D= (output==NULL) ? stdout : fopen(output,"wb");
   if (D==NULL) cannot("open");
   lua_lock(L);
-  luaU_dump(L,f,writer,D,stripping);
+  luaU_dump(L,f,luac_writer,D,stripping);
   lua_unlock(L);
   if (ferror(D)) cannot("write");
   if (fclose(D)) cannot("close");