Improve processing of first-line comments by LuaParser

Author: James Roseborough

grammar/Lua52.jj

@@ -133,7 +133,6 @@ TOKEN :
 | < #QUOTED: <DECIMAL> | <UNICODE> | <CHAR> >
 | < #DECIMAL:   "\\" ["0"-"9"] (["0"-"9"])? (["0"-"9"])? >
 | < DBCOLON: "::" >
-| < SHEBANG: "#" ["!"," "] (~["\n","\r"])* (<LF>)? >
 | < #UNICODE: "\\" "u" <HEXDIGIT> <HEXDIGIT> <HEXDIGIT> <HEXDIGIT> >
 | < #CHAR: "\\" (~[]) >
 | < #LF: ("\n" | "\r" | "\r\n") >
@@ -143,7 +142,7 @@ TOKEN :
 void Chunk():
 {}
 {
-	( <SHEBANG> )? Block() <EOF>
+	( "#" { token_source.SwitchTo(IN_COMMENT); } )? Block() <EOF>
 }
 
 void Block():

grammar/LuaParser.jj

@@ -157,7 +157,6 @@ TOKEN :
 | < #QUOTED: <DECIMAL> | <UNICODE> | <CHAR> >
 | < #DECIMAL:   "\\" ["0"-"9"] (["0"-"9"])? (["0"-"9"])? >
 | < DBCOLON: "::" >
-| < SHEBANG: "#" ["!"," "] (~["\n","\r"])* (<LF>)? >
 | < #UNICODE: "\\" "u" <HEXDIGIT> <HEXDIGIT> <HEXDIGIT> <HEXDIGIT> >
 | < #CHAR: "\\" (~[]) >
 | < #LF: ("\n" | "\r" | "\r\n") >
@@ -169,7 +168,7 @@ Chunk Chunk():
 	Block b;
 }
 {
-	( <SHEBANG> )? b=Block() <EOF> { return new Chunk(b); }
+	( "#" { token_source.SwitchTo(IN_COMMENT); } )? b=Block() <EOF> { return new Chunk(b); }
 }
 
 Block Block():

src/jse/org/luaj/vm2/parser/LuaParser.java

@@ -27,15 +27,16 @@ private static Exp.FuncCall assertfunccall(Exp.PrimaryExp pe) throws ParseExcept
   final public Chunk Chunk() throws ParseException {
         Block b;
     switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
-    case SHEBANG:
-      jj_consume_token(SHEBANG);
+    case 69:
+      jj_consume_token(69);
+                token_source.SwitchTo(IN_COMMENT);
       break;
     default:
       ;
     }
     b = Block();
     jj_consume_token(0);
-                                         {if (true) return new Chunk(b);}
+                                                                          {if (true) return new Chunk(b);}
     throw new Error("Missing return statement in function");
   }
 
@@ -264,11 +265,11 @@ final public Stat ReturnStat() throws ParseException {
     case NUMBER:
     case STRING:
     case CHARSTRING:
+    case 69:
     case 75:
     case 79:
     case 80:
     case 83:
-    case 95:
       el = ExpList();
       break;
     default:
@@ -472,11 +473,11 @@ final public FuncArgs FuncArgs() throws ParseException {
       case NUMBER:
       case STRING:
       case CHARSTRING:
+      case 69:
       case 75:
       case 79:
       case 80:
       case 83:
-      case 95:
         el = ExpList();
         break;
       default:
@@ -665,8 +666,8 @@ final public Exp Exp() throws ParseException {
       e = SimpleExp();
       break;
     case NOT:
+    case 69:
     case 83:
-    case 95:
       op = Unop();
       s = Exp();
                                              e=Exp.unaryexp(op,s);
@@ -764,12 +765,12 @@ final public TableConstructor TableConstructor() throws ParseException {
     case NUMBER:
     case STRING:
     case CHARSTRING:
+    case 69:
     case 75:
     case 77:
     case 79:
     case 80:
     case 83:
-    case 95:
       fl = FieldList();
                               tc.fields=fl;
       break;
@@ -843,11 +844,11 @@ final public TableField Field() throws ParseException {
         case NUMBER:
         case STRING:
         case CHARSTRING:
+        case 69:
         case 75:
         case 79:
         case 80:
         case 83:
-        case 95:
           rhs = Exp();
                                                   {if (true) return TableField.listField(rhs);}
           break;
@@ -953,8 +954,8 @@ final public int Unop() throws ParseException {
       jj_consume_token(NOT);
                 {if (true) return Lua.OP_NOT;}
       break;
-    case 95:
-      jj_consume_token(95);
+    case 69:
+      jj_consume_token(69);
                 {if (true) return Lua.OP_LEN;}
       break;
     default:
@@ -1006,6 +1007,11 @@ private boolean jj_2_7(int xla) {
     catch(LookaheadSuccess ls) { return true; }
   }
 
+  private boolean jj_3R_28() {
+    if (jj_scan_token(91)) return true;
+    return false;
+  }
+
   private boolean jj_3R_27() {
     if (jj_scan_token(90)) return true;
     return false;
@@ -1396,7 +1402,7 @@ private boolean jj_3R_12() {
   }
 
   private boolean jj_3R_55() {
-    if (jj_scan_token(95)) return true;
+    if (jj_scan_token(69)) return true;
     return false;
   }
 
@@ -1514,11 +1520,6 @@ private boolean jj_3R_29() {
     return false;
   }
 
-  private boolean jj_3R_28() {
-    if (jj_scan_token(91)) return true;
-    return false;
-  }
-
   /** Generated Token Manager. */
   public LuaParserTokenManager token_source;
   SimpleCharStream jj_input_stream;

src/jse/org/luaj/vm2/parser/LuaParserConstants.java

@@ -107,13 +107,11 @@ public interface LuaParserConstants {
   /** RegularExpression Id. */
   int DBCOLON = 65;
   /** RegularExpression Id. */
-  int SHEBANG = 66;
+  int UNICODE = 66;
   /** RegularExpression Id. */
-  int UNICODE = 67;
+  int CHAR = 67;
   /** RegularExpression Id. */
-  int CHAR = 68;
-  /** RegularExpression Id. */
-  int LF = 69;
+  int LF = 68;
 
   /** Lexical state. */
   int DEFAULT = 0;
@@ -208,10 +206,10 @@ public interface LuaParserConstants {
     "<QUOTED>",
     "<DECIMAL>",
     "\"::\"",
-    "<SHEBANG>",
     "<UNICODE>",
     "<CHAR>",
     "<LF>",
+    "\"#\"",
     "\";\"",
     "\"=\"",
     "\",\"",
@@ -237,7 +235,6 @@ public interface LuaParserConstants {
     "\">=\"",
     "\"==\"",
     "\"~=\"",
-    "\"#\"",
   };
 
 }