Use Ragel to distinguish block args from | ops.

Author: smashwilson

examples/input.seg

@@ -1,5 +1,4 @@
-# This is a bunch of statements!
-
 {
+  |a, b|
   3 + 4
 }

src/lexer.h

@@ -11,4 +11,10 @@
  */
 seg_statementlist_node *seg_parse(char *content, off_t length, seg_options *opts);
 
+/* The initial size of the Ragel stack. */
+#define RAGEL_INIT_STACK_SIZE 5
+
+/* The amount by which the Ragel stack will be increased or decreased. */
+#define RAGEL_STACK_INCR 10
+
 #endif

src/lexer.rl

@@ -3,6 +3,7 @@
 #include <string.h>
 #include <sys/types.h>
 
+#include "lexer.h"
 #include "ast.h"
 #include "token.h"
 #include "segment.h"
@@ -43,16 +44,30 @@ static void report(const char *name, const char *ts, const char *te) {
   control = [(){};=.|,%@];
   op = [&|+\-*/%^];
 
-  nonws = ^whitespace;
+  nonws = ^whitespace & [^\r\n];
   noncontrol = ^control;
   nonop = ^op;
+  nonnumeric = [^0-9];
 
-  iboundary = noncontrol & nonop & nonws;
+  iboundary = nonws & noncontrol & nonnumeric;
   imiddle = nonws;
 
-  identifier = iboundary imiddle iboundary | iboundary;
+  identifier = iboundary imiddle* iboundary | iboundary;
   symbol = ':' identifier | ':' string;
 
+  blockargs := |*
+    comment;
+    whitespace;
+    [\r\n];
+
+    identifier => { CAPTURE(IDENTIFIER); };
+    ',' => { EMPTY(COMMA); };
+    '|' => {
+      EMPTY(BAR);
+      fret;
+    };
+  *|;
+
   main := |*
     comment;
 
@@ -71,9 +86,13 @@ static void report(const char *name, const char *ts, const char *te) {
     '\n' => { EMPTY(NEWLINE); };
     '=' => { EMPTY(ASSIGNMENT); };
     '.' => { EMPTY(PERIOD); };
-    '|' => { EMPTY(BAR); };
     ',' => { EMPTY(COMMA); };
 
+    '|' => {
+      EMPTY(BAR);
+      fcall blockargs;
+    };
+
     identifier '(' => { EMPTY(METHODNAME); };
 
     identifier ':' => { EMPTY(KEYWORD); };
@@ -94,6 +113,26 @@ static void report(const char *name, const char *ts, const char *te) {
 
     whitespace;
   *|;
+
+  # Stack management
+
+  prepush {
+    if (top >= stack_size) {
+      if (opts->verbose) {
+        printf("Growing stack from %d to %d.\n", stack_size, stack_size + RAGEL_STACK_INCR);
+      }
+      stack = realloc(stack, stack_size + RAGEL_STACK_INCR);
+    }
+  }
+
+  postpop {
+    if (stack_size - top >= RAGEL_STACK_INCR) {
+      if (opts->verbose) {
+        printf("Shrinking stack from %d to %d.\n", stack_size, stack_size - RAGEL_STACK_INCR);
+      }
+      stack = realloc(stack, stack_size - RAGEL_STACK_INCR);
+    }
+  }
 }%%
 /* Syntax Highlighting */
 
@@ -102,12 +141,15 @@ static void report(const char *name, const char *ts, const char *te) {
 seg_statementlist_node *seg_parse(char *content, off_t length, seg_options *opts)
 {
   /* Variables used by Ragel. */
-  int cs, act;
+  int cs, act, top;
   char *ts, *te;
   char *p = content;
   const char *pe = content + length;
   const char *eof = pe;
 
+  int *stack = malloc(sizeof(int) * RAGEL_INIT_STACK_SIZE);
+  int stack_size = RAGEL_INIT_STACK_SIZE;
+
   /* Parser state */
   int lexer_error = 0;
   void *parser = ParseAlloc(malloc);